diff --git a/.github/workflows/backend-check.yml b/.github/workflows/backend-check.yml index 901a30cca6..7fc0874564 100644 --- a/.github/workflows/backend-check.yml +++ b/.github/workflows/backend-check.yml @@ -53,7 +53,7 @@ jobs: timeout-minutes: 16 run: | mkdir -p fake_frontend_build - FRONTEND_BUILD_DIR=$(pwd)/fake_frontend_build SQLX_OFFLINE=true cargo check --all-features + FRONTEND_BUILD_DIR=$(pwd)/fake_frontend_build SQLX_OFFLINE=true cargo check --features $(./all_features_oss.sh) check_ee: runs-on: ubicloud-standard-8 diff --git a/.github/workflows/build-publish-rh-image.yml b/.github/workflows/build-publish-rh-image.yml index b919de6e18..848c30c7f6 100644 --- a/.github/workflows/build-publish-rh-image.yml +++ b/.github/workflows/build-publish-rh-image.yml @@ -64,7 +64,7 @@ jobs: platforms: linux/amd64 push: true build-args: | - features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,license,otel,http_trigger,zip,oauth2,kafka,sqs_trigger,nats,postgres_trigger,gcp_trigger,mqtt_trigger,websocket,smtp,static_frontend,all_languages,deno_core,mcp + features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,license,otel,http_trigger,zip,oauth2,kafka,sqs_trigger,nats,postgres_trigger,gcp_trigger,mqtt_trigger,websocket,smtp,static_frontend,all_languages,deno_core,mcp,private secrets: | rh_username=${{ secrets.RH_USERNAME }} rh_password=${{ secrets.RH_PASSWORD }} @@ -81,7 +81,7 @@ jobs: platforms: linux/arm64 push: true build-args: | - features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,license,otel,http_trigger,zip,oauth2,kafka,sqs_trigger,nats,postgres_trigger,gcp_trigger,mqtt_trigger,websocket,smtp,static_frontend,all_languages,deno_core,mcp + features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,license,otel,http_trigger,zip,oauth2,kafka,sqs_trigger,nats,postgres_trigger,gcp_trigger,mqtt_trigger,websocket,smtp,static_frontend,all_languages,deno_core,mcp,private secrets: | rh_username=${{ secrets.RH_USERNAME }} rh_password=${{ secrets.RH_PASSWORD }} diff --git a/.github/workflows/build_windows_worker_.yml b/.github/workflows/build_windows_worker_.yml index ac1476d76e..738b2a9224 100644 --- a/.github/workflows/build_windows_worker_.yml +++ b/.github/workflows/build_windows_worker_.yml @@ -51,7 +51,7 @@ jobs: $env:OPENSSL_DIR="${Env:VCPKG_INSTALLATION_ROOT}\installed\x64-windows-static" mkdir frontend/build && cd backend New-Item -Path . -Name "windmill-api/openapi-deref.yaml" -ItemType "File" -Force - cargo build --release --features=enterprise,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,tantivy,license,http_trigger,zip,oauth2,kafka,nats,sqs_trigger,postgres_trigger,gcp_trigger,mqtt_trigger,websocket,smtp,static_frontend,all_languages,mcp + cargo build --release --features=enterprise,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,tantivy,license,http_trigger,zip,oauth2,kafka,nats,sqs_trigger,postgres_trigger,gcp_trigger,mqtt_trigger,websocket,smtp,static_frontend,all_languages_windows,mcp,private - name: Rename binary with corresponding architecture run: | Rename-Item -Path ".\backend\target\release\windmill.exe" -NewName "windmill-ee.exe" diff --git a/.github/workflows/create-docs.yml b/.github/workflows/create-docs.yml index 209883d24c..2d264f9755 100644 --- a/.github/workflows/create-docs.yml +++ b/.github/workflows/create-docs.yml @@ -12,12 +12,12 @@ jobs: - name: Check organization membership id: check-membership env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + ORG_ACCESS_TOKEN: ${{ secrets.ORG_ACCESS_TOKEN }} COMMENTER: ${{ github.event.comment.user.login }} run: | ORG="windmill-labs" STATUS=$(curl -s -o /dev/null -w "%{http_code}" \ - -H "Authorization: token $GH_TOKEN" \ + -H "Authorization: token $ORG_ACCESS_TOKEN" \ -H "Accept: application/vnd.github+json" \ -H "X-GitHub-Api-Version: 2022-11-28" \ "https://api.github.com/orgs/$ORG/members/$COMMENTER") diff --git a/.github/workflows/discord-notification.yml b/.github/workflows/discord-notification.yml index 3ffec78a60..012c406cef 100644 --- a/.github/workflows/discord-notification.yml +++ b/.github/workflows/discord-notification.yml @@ -17,8 +17,11 @@ jobs: PR_AUTHOR: ${{ github.event.pull_request.user.login }} PR_STATUS: "opened" PR_NUMBER: ${{ github.event.pull_request.number }} + DISCORD_CHANNEL_ID: "1372204995868491786" + DISCORD_GUILD_ID: "930051556043276338" secrets: DISCORD_WEBHOOK_URL: ${{ secrets.DISCORD_PR_REVIEWS_WEBHOOK }} + DISCORD_BOT_TOKEN: ${{ secrets.DISCORD_AI_BOT_TOKEN }} merge_success_emoji: if: github.event.pull_request.merged == true diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml index 216d7cefea..5425ccb7e1 100644 --- a/.github/workflows/docker-image.yml +++ b/.github/workflows/docker-image.yml @@ -92,7 +92,7 @@ jobs: platforms: linux/amd64,linux/arm64 push: true build-args: | - features=embedding,parquet,openidconnect,jemalloc,license,http_trigger,zip,oauth2,dind,postgres_trigger,mqtt_trigger,websocket,smtp,static_frontend,agent_worker_server,all_languages,deno_core,mcp + features=embedding,parquet,openidconnect,jemalloc,license,http_trigger,zip,oauth2,dind,postgres_trigger,mqtt_trigger,websocket,smtp,static_frontend,agent_worker_server,all_languages,deno_core,mcp,private tags: | ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ env.DEV_SHA }} ${{ steps.meta-public.outputs.tags }} @@ -154,7 +154,7 @@ jobs: platforms: linux/amd64,linux/arm64 push: true build-args: | - features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,agent_worker_server,tantivy,license,http_trigger,zip,oauth2,kafka,sqs_trigger,nats,otel,dind,postgres_trigger,mqtt_trigger,gcp_trigger,websocket,smtp,static_frontend,all_languages,deno_core,mcp + features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,agent_worker_server,tantivy,license,http_trigger,zip,oauth2,kafka,sqs_trigger,nats,otel,dind,postgres_trigger,mqtt_trigger,gcp_trigger,websocket,smtp,static_frontend,all_languages,private,deno_core,mcp tags: | ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:${{ env.DEV_SHA }} ${{ steps.meta-ee-public.outputs.tags }} diff --git a/.github/workflows/helmchart_on_release.yml b/.github/workflows/helmchart_on_release.yml index 1d63cbc326..eeae218d5d 100644 --- a/.github/workflows/helmchart_on_release.yml +++ b/.github/workflows/helmchart_on_release.yml @@ -13,7 +13,7 @@ jobs: uses: actions/checkout@v3 with: repository: windmill-labs/windmill-helm-charts - token: ${{ secrets.DOCS_TOKEN }} + token: ${{ secrets.HELM_CHART_TOKEN }} - name: Get version id: get_version @@ -57,7 +57,7 @@ jobs: - name: Create PR env: - GH_TOKEN: ${{ secrets.DOCS_TOKEN }} + GH_TOKEN: ${{ secrets.HELM_CHART_TOKEN }} run: | gh pr create \ --title "helm: bump version to ${{ env.VERSION }}" \ diff --git a/.github/workflows/publish_windows_worker.yml b/.github/workflows/publish_windows_worker.yml index 057a858881..e9a8c77e87 100644 --- a/.github/workflows/publish_windows_worker.yml +++ b/.github/workflows/publish_windows_worker.yml @@ -53,7 +53,7 @@ jobs: $env:OPENSSL_DIR="${Env:VCPKG_INSTALLATION_ROOT}\installed\x64-windows-static" mkdir frontend/build && cd backend New-Item -Path . -Name "windmill-api/openapi-deref.yaml" -ItemType "File" -Force - cargo build --release --features=enterprise,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,tantivy,license,http_trigger,zip,oauth2,kafka,sqs_trigger,nats,postgres_trigger,mqtt_trigger,gcp_trigger,websocket,smtp,static_frontend,all_languages,mcp + cargo build --release --features=enterprise,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,tantivy,license,http_trigger,zip,oauth2,kafka,sqs_trigger,nats,postgres_trigger,mqtt_trigger,gcp_trigger,websocket,smtp,static_frontend,all_languages_windows,mcp,private - name: Rename binary with corresponding architecture run: | Rename-Item -Path ".\backend\target\release\windmill.exe" -NewName "windmill-ee.exe" diff --git a/.github/workflows/rust_on_release.yml b/.github/workflows/rust_on_release.yml new file mode 100644 index 0000000000..5594cc08bb --- /dev/null +++ b/.github/workflows/rust_on_release.yml @@ -0,0 +1,19 @@ +name: Publish rust-client to crates.io on release +on: + push: + tags: + - "v*" + workflow_dispatch: + +jobs: + build_rust_and_publish_to_crates_io: + runs-on: ubicloud-standard-8 + steps: + - uses: actions/checkout@v4 + - uses: cachix/install-nix-action@v20 + with: + extra_nix_config: | + experimental-features = nix-command flakes + - run: cd rust-client && nix develop ../ --command ./dev.nu --check --publish + env: + CRATES_IO_TOKEN: ${{ secrets.CRATES_IO_TOKEN }} diff --git a/.github/workflows/shareable-discord-notification.yml b/.github/workflows/shareable-discord-notification.yml index cb6dbf055d..1b7936330f 100644 --- a/.github/workflows/shareable-discord-notification.yml +++ b/.github/workflows/shareable-discord-notification.yml @@ -38,24 +38,45 @@ jobs: - name: Send Discord notification and start a thread env: WEBHOOK_URL: ${{ secrets.DISCORD_WEBHOOK_URL }} + BOT_TOKEN: ${{ secrets.DISCORD_BOT_TOKEN }} + CHANNEL_ID: ${{ inputs.DISCORD_CHANNEL_ID }} + GUILD_ID: ${{ inputs.DISCORD_GUILD_ID }} PR_TITLE: ${{ inputs.PR_TITLE }} PR_NUMBER: ${{ inputs.PR_NUMBER }} PR_URL: ${{ inputs.PR_URL }} PR_AUTHOR: ${{ inputs.PR_AUTHOR }} run: | - payload=$(jq -n \ - --arg content "${PR_URL}" \ - --arg thread "#${PR_NUMBER}: $PR_TITLE by \`${PR_AUTHOR}\`" \ - '{ - content: $content, - thread_name: $thread, - auto_archive_duration: 10080 - }' - ) - curl -H "Content-Type: application/json" \ - -X POST \ - -d "$payload" \ - "$WEBHOOK_URL" + # Check if thread already exists + thread_exists=false + if threads=$(curl -s -H "Authorization: Bot $BOT_TOKEN" "https://discord.com/api/v10/guilds/${GUILD_ID}/threads/active"); then + if thread_id=$(echo "$threads" | jq -r --arg cid "$CHANNEL_ID" --arg pref "#${PR_NUMBER}:" '.threads[] | select(.parent_id == $cid and (.name | startswith($pref))) | .id' 2>/dev/null); then + if [ -n "$thread_id" ]; then + thread_exists=true + echo "Thread already exists, skipping creation" + fi + fi + else + echo "Failed to check for existing threads, will create new thread" + fi + + # Create thread if it doesn't exist or if check failed + if [ "$thread_exists" = false ]; then + echo "Creating new thread" + THREAD_TITLE="#${PR_NUMBER}: ${PR_TITLE} by \`${PR_AUTHOR}\`" + payload=$(jq -n \ + --arg content "${PR_URL}" \ + --arg thread "${THREAD_TITLE:0:99}" \ + '{ + content: $content, + thread_name: $thread, + auto_archive_duration: 10080 + }' + ) + curl -H "Content-Type: application/json" \ + -X POST \ + -d "$payload" \ + "$WEBHOOK_URL" + fi merge_success_emoji: runs-on: ubuntu-latest diff --git a/CHANGELOG.md b/CHANGELOG.md index 94351573de..a496bda1b5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,111 @@ # Changelog +## [1.498.0](https://github.com/windmill-labs/windmill/compare/v1.497.2...v1.498.0) (2025-06-17) + + +### Features + +* use provider api to list available AI models in workspace settings ([#5947](https://github.com/windmill-labs/windmill/issues/5947)) ([7490e88](https://github.com/windmill-labs/windmill/commit/7490e883d747a7f65b2fefd3ec14b1cfc3d9bbd4)) +* windmill http triggers and webhooks to openapi spec ([#5918](https://github.com/windmill-labs/windmill/issues/5918)) ([aba8c01](https://github.com/windmill-labs/windmill/commit/aba8c01d7f44ba4be369a3c711be9e156d6bf215)) + +## [1.497.2](https://github.com/windmill-labs/windmill/compare/v1.497.1...v1.497.2) (2025-06-17) + + +### Bug Fixes + +* always rm containers in docker mode ([38eb71b](https://github.com/windmill-labs/windmill/commit/38eb71bdf55ee2f606d1d2ad2e987d5af16d88c0)) +* flow steps use their tags if any specific when used as subflow ([26bec05](https://github.com/windmill-labs/windmill/commit/26bec054a3447a91c5d5f56d8b98717c06496087)) + +## [1.497.1](https://github.com/windmill-labs/windmill/compare/v1.497.0...v1.497.1) (2025-06-16) + + +### Bug Fixes + +* fix mcp server initialization ([1c6a7c8](https://github.com/windmill-labs/windmill/commit/1c6a7c8cd0bd8396f158e3cb0583b927ce957f12)) + +## [1.497.0](https://github.com/windmill-labs/windmill/compare/v1.496.3...v1.497.0) (2025-06-16) + + +### Features + +* add api tools to ai chat ([#5921](https://github.com/windmill-labs/windmill/issues/5921)) ([f7a83c0](https://github.com/windmill-labs/windmill/commit/f7a83c03c12b8ae70179fb228e0e2391b6ea2858)) +* **backend:** use streamable http in favor of sse for MCP ([#5910](https://github.com/windmill-labs/windmill/issues/5910)) ([d47c078](https://github.com/windmill-labs/windmill/commit/d47c078bb5ab86d82d9cbbce3c55c89c0c20d809)) +* better graph layout algorithm + migrate to svelte 5 almost everywhere + xyflow 1.0 ([23920ae](https://github.com/windmill-labs/windmill/commit/23920aee84fdca4a557a34ff2d66a0bb7bdca605)) +* fill runnable inputs with AI chat ([#5887](https://github.com/windmill-labs/windmill/issues/5887)) ([b4a6a7e](https://github.com/windmill-labs/windmill/commit/b4a6a7e72429617d420af85a9de35bb13adfc6fb)) +* **go:** local go.mod ([#5929](https://github.com/windmill-labs/windmill/issues/5929)) ([0b89260](https://github.com/windmill-labs/windmill/commit/0b89260540b307c6d614ca4275dd038fbfdac33c)) +* multiple azure models support ([#5920](https://github.com/windmill-labs/windmill/issues/5920)) ([f412ede](https://github.com/windmill-labs/windmill/commit/f412ede6ed48e9a492f39582ac70a5584477529e)) +* **rust:** add rust sdk ([#5909](https://github.com/windmill-labs/windmill/issues/5909)) ([332f66e](https://github.com/windmill-labs/windmill/commit/332f66e3483abbeacd4e7c1b74c94c5265314882)) + + +### Bug Fixes + +* ai chat tooltip + user settings autocomplete issue ([#5917](https://github.com/windmill-labs/windmill/issues/5917)) ([6f907c7](https://github.com/windmill-labs/windmill/commit/6f907c79b4cf6279bd52e35a3ee96e0d021422f5)) +* audit logs for token refresh + consider refresh for active users ([#5930](https://github.com/windmill-labs/windmill/issues/5930)) ([cf2d09e](https://github.com/windmill-labs/windmill/commit/cf2d09e7a8c5d2472af0d483689c3fcfa2976117)) +* fix input with wrong height on first render ([#5935](https://github.com/windmill-labs/windmill/issues/5935)) ([1a6283b](https://github.com/windmill-labs/windmill/commit/1a6283b42a6a514ab2e05160855cdc0f70b61d0e)) +* flow step missing input warnings ([#5916](https://github.com/windmill-labs/windmill/issues/5916)) ([f077849](https://github.com/windmill-labs/windmill/commit/f077849b8f7c1916fd420e85b4844a5c5e93a139)) +* **frontend:** use correct kind for flow insert module btn ([#5938](https://github.com/windmill-labs/windmill/issues/5938)) ([17c8c8a](https://github.com/windmill-labs/windmill/commit/17c8c8a5616ab8656799cea3fc5bc7cfaedc4995)) + +## [1.496.3](https://github.com/windmill-labs/windmill/compare/v1.496.2...v1.496.3) (2025-06-09) + + +### Bug Fixes + +* improve concurrent job parallelism performance ([e8836a3](https://github.com/windmill-labs/windmill/commit/e8836a393a872bb91e68ba0037681caf24149470)) +* Prioritize diff contexts in script mode for ai chat ([#5888](https://github.com/windmill-labs/windmill/issues/5888)) ([a47939d](https://github.com/windmill-labs/windmill/commit/a47939d13c30e2d4b41efd539f845959174d4fb1)) + +## [1.496.2](https://github.com/windmill-labs/windmill/compare/v1.496.1...v1.496.2) (2025-06-07) + + +### Bug Fixes + +* add clearable by default for select ([#5900](https://github.com/windmill-labs/windmill/issues/5900)) ([b44b9c1](https://github.com/windmill-labs/windmill/commit/b44b9c1b82116ad5487af95d1f78226d56c75179)) + +## [1.496.1](https://github.com/windmill-labs/windmill/compare/v1.496.0...v1.496.1) (2025-06-07) + + +### Bug Fixes + +* never consider minor version for global site packages ([#5893](https://github.com/windmill-labs/windmill/issues/5893)) ([22b2f49](https://github.com/windmill-labs/windmill/commit/22b2f4988db9314f2403508933d0aa932187c668)) + +## [1.496.0](https://github.com/windmill-labs/windmill/compare/v1.495.1...v1.496.0) (2025-06-06) + + +### Features + +* generate http route triggers from openapi spec ([#5857](https://github.com/windmill-labs/windmill/issues/5857)) ([5713483](https://github.com/windmill-labs/windmill/commit/571348377b73d54b4d2a1c5775ab00b247b01910)) + + +### Bug Fixes + +* allow fileupload drag and drop in edit mode on full component without triggering file picker ([#5889](https://github.com/windmill-labs/windmill/issues/5889)) ([9ae3212](https://github.com/windmill-labs/windmill/commit/9ae3212a1e0f88a8297bf41ab53e3c1be4bcc56c)) +* **python:** account instance version when cli deploy and local lockfile ([#5894](https://github.com/windmill-labs/windmill/issues/5894)) ([ec552d5](https://github.com/windmill-labs/windmill/commit/ec552d5ef6fdb5e824e453f196f9cf16629ee2ea)) +* use full client side js library for route gen from openapi ([#5891](https://github.com/windmill-labs/windmill/issues/5891)) ([3c3fdbd](https://github.com/windmill-labs/windmill/commit/3c3fdbdf26a9581b815210839b91ebdedb924093)) + +## [1.495.0](https://github.com/windmill-labs/windmill/compare/v1.494.0...v1.495.0) (2025-06-05) + + +### Features + +* Add ask mode to AI chat ([#5878](https://github.com/windmill-labs/windmill/issues/5878)) ([67ab469](https://github.com/windmill-labs/windmill/commit/67ab46990ad0c9fad810a64c54297419c6151c79)) +* add navigator mode to AIChat and unify UI ([#5859](https://github.com/windmill-labs/windmill/issues/5859)) ([cbba829](https://github.com/windmill-labs/windmill/commit/cbba8297cd4c1caa21b96a8422bbbd5c306b8398)) +* ai flow chat ([#5842](https://github.com/windmill-labs/windmill/issues/5842)) ([68ebf66](https://github.com/windmill-labs/windmill/commit/68ebf667d5c0bc306329d0b55a3cc59e5b4862cb)) +* ai prompts improvements + o3/o4 support ([#5862](https://github.com/windmill-labs/windmill/issues/5862)) ([825422c](https://github.com/windmill-labs/windmill/commit/825422c48456b2c9b230e1a35914b3fbf7d1e836)) +* connect fix btn in flow editor to ai chat ([#5863](https://github.com/windmill-labs/windmill/issues/5863)) ([6247d15](https://github.com/windmill-labs/windmill/commit/6247d159ce25ae13f6fbc5c105df88305ce29451)) +* fix backward compatibility pg 14 for postgres trigger ([#5851](https://github.com/windmill-labs/windmill/issues/5851)) ([4cbcbdb](https://github.com/windmill-labs/windmill/commit/4cbcbdb960b469acf773d3943128b6c7d0dcb0b8)) +* ssh repl like direct to workers hosts machine ([#5809](https://github.com/windmill-labs/windmill/issues/5809)) ([f252657](https://github.com/windmill-labs/windmill/commit/f2526571a3614156b2b1e5cc91b15d0c57565d99)) +* use rust-postgres client instead of sqlx for postgres trigger ([#5853](https://github.com/windmill-labs/windmill/issues/5853)) ([39dbd64](https://github.com/windmill-labs/windmill/commit/39dbd646b9683e0ad8de047cca786ae468759e77)) + + +### Bug Fixes + +* broken event dispatch for simpleditor ([#5879](https://github.com/windmill-labs/windmill/issues/5879)) ([df4992a](https://github.com/windmill-labs/windmill/commit/df4992a9295ed188c2a2cb0a5dfd3e33ae2e2dcb)) +* cannot parse INSTANCE_PYTHON_VERSION ([#5874](https://github.com/windmill-labs/windmill/issues/5874)) ([a0b302d](https://github.com/windmill-labs/windmill/commit/a0b302d2c58d4245260376cf280bc866be91717c)) +* fix regex that extract workspaces from custom tags ([#5876](https://github.com/windmill-labs/windmill/issues/5876)) ([1551dc8](https://github.com/windmill-labs/windmill/commit/1551dc8af22f6ea41f68290ace4c58f936c47745)) +* nit ai flow prompt ([#5867](https://github.com/windmill-labs/windmill/issues/5867)) ([3e769f0](https://github.com/windmill-labs/windmill/commit/3e769f0c591b80138b3a356d147228675756452f)) +* **python:** assign PATCH version to python runtime only when needed ([#5866](https://github.com/windmill-labs/windmill/issues/5866)) ([50a5c1f](https://github.com/windmill-labs/windmill/commit/50a5c1f56a7e45882fa0095203de709571e149bb)) +* remove duplicate tools from script ai chat ([#5880](https://github.com/windmill-labs/windmill/issues/5880)) ([fe4a767](https://github.com/windmill-labs/windmill/commit/fe4a767df0e6f46fd0c0fd21b4116c7375978bf9)) +* replace crypto.randomUUID with generateRandomString for HTTP compatibility ([#5849](https://github.com/windmill-labs/windmill/issues/5849)) ([64f35d0](https://github.com/windmill-labs/windmill/commit/64f35d050fb0d1008ce7142fd62d500845e62c4a)), closes [#5847](https://github.com/windmill-labs/windmill/issues/5847) + ## [1.494.0](https://github.com/windmill-labs/windmill/compare/v1.493.4...v1.494.0) (2025-05-31) diff --git a/CLAUDE.md b/CLAUDE.md index 00d81d1b48..5a6512b371 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1,3 +1,3 @@ To have an overview of what this app does, see @.cursor/rules/windmill-overview.mdc -For backend modifications, follow the rules mentioned here @.cursor/rules/rust-best-practices.mdc +For backend modifications, follow the rules mentioned here @.cursor/rules/rust-best-practices.mdc. You also have access to a summarized version of the database schema here @backend/summarized_schema.txt For frontend modifications, follow the rules mentioned here @.cursor/rules/svelte5-best-practices.mdc diff --git a/README.md b/README.md index 27a5aaf296..c86977bf72 100644 --- a/README.md +++ b/README.md @@ -367,10 +367,11 @@ you to have it being synced automatically everyday. ## Run a local dev setup +Using [Nix](./frontend/README_DEV.md#nix) (Recommended). + See the [./frontend/README_DEV.md](./frontend/README_DEV.md) file for all running options. -Using [Nix](./frontend/README_DEV.md#nix). ### only Frontend diff --git a/backend/.gitignore b/backend/.gitignore index f75b9d1eaa..2a3262acac 100644 --- a/backend/.gitignore +++ b/backend/.gitignore @@ -6,4 +6,5 @@ tracing.folded heaptrack* index/ windmill-api/openapi-*.* -.duckdb/* \ No newline at end of file +.duckdb/* +*ee.rs \ No newline at end of file diff --git a/backend/.ignore b/backend/.ignore new file mode 100644 index 0000000000..3173520127 --- /dev/null +++ b/backend/.ignore @@ -0,0 +1 @@ +!*ee.rs \ No newline at end of file diff --git a/backend/.sqlx/query-0937e25e89959447e7cb1816c112bbc4718cbb8ad6e2f13eae6b14f129d12936.json b/backend/.sqlx/query-0937e25e89959447e7cb1816c112bbc4718cbb8ad6e2f13eae6b14f129d12936.json index 7d24993172..152ae38fdc 100644 --- a/backend/.sqlx/query-0937e25e89959447e7cb1816c112bbc4718cbb8ad6e2f13eae6b14f129d12936.json +++ b/backend/.sqlx/query-0937e25e89959447e7cb1816c112bbc4718cbb8ad6e2f13eae6b14f129d12936.json @@ -61,7 +61,8 @@ "csharp", "oracledb", "nu", - "java" + "java", + "duckdb" ] } } diff --git a/backend/.sqlx/query-0de53d764bbcf44d76cd8e47f7b2cd49e4632f03f16ee6f34d9a2c0842890f05.json b/backend/.sqlx/query-0de53d764bbcf44d76cd8e47f7b2cd49e4632f03f16ee6f34d9a2c0842890f05.json new file mode 100644 index 0000000000..fd8f30417f --- /dev/null +++ b/backend/.sqlx/query-0de53d764bbcf44d76cd8e47f7b2cd49e4632f03f16ee6f34d9a2c0842890f05.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT \n path\n FROM\n flow\n WHERE\n path ~ ANY($1) AND\n workspace_id = $2\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "path", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "TextArray", + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "0de53d764bbcf44d76cd8e47f7b2cd49e4632f03f16ee6f34d9a2c0842890f05" +} diff --git a/backend/.sqlx/query-19f0ccadd3ee44719a781ea0d73ea4e45f5b2c3d5c0aa5dbecf9ea9838881b74.json b/backend/.sqlx/query-19f0ccadd3ee44719a781ea0d73ea4e45f5b2c3d5c0aa5dbecf9ea9838881b74.json new file mode 100644 index 0000000000..62c47f2503 --- /dev/null +++ b/backend/.sqlx/query-19f0ccadd3ee44719a781ea0d73ea4e45f5b2c3d5c0aa5dbecf9ea9838881b74.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "CREATE INDEX CONCURRENTLY IF NOT EXISTS v2_job_queue_suspend ON v2_job_queue (workspace_id, suspend) WHERE suspend > 0;", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "19f0ccadd3ee44719a781ea0d73ea4e45f5b2c3d5c0aa5dbecf9ea9838881b74" +} diff --git a/backend/.sqlx/query-1d87f41fd1abb9361d795a899120e6b77e24bf5a9044fdc5284d0d7f1e14eafa.json b/backend/.sqlx/query-1d87f41fd1abb9361d795a899120e6b77e24bf5a9044fdc5284d0d7f1e14eafa.json deleted file mode 100644 index 6a79013c0b..0000000000 --- a/backend/.sqlx/query-1d87f41fd1abb9361d795a899120e6b77e24bf5a9044fdc5284d0d7f1e14eafa.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "INSERT INTO usage (id, is_workspace, month_, usage) \n VALUES ($1, TRUE, EXTRACT(YEAR FROM current_date) * 12 + EXTRACT(MONTH FROM current_date), $2) \n ON CONFLICT (id, is_workspace, month_) DO UPDATE SET usage = usage.usage + $2", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Varchar", - "Int4" - ] - }, - "nullable": [] - }, - "hash": "1d87f41fd1abb9361d795a899120e6b77e24bf5a9044fdc5284d0d7f1e14eafa" -} diff --git a/backend/.sqlx/query-2bf5f7f2cf9d85a5d23e5db2f7616fb41fece9b3d46fde2d546d70b46f9008e3.json b/backend/.sqlx/query-2bf5f7f2cf9d85a5d23e5db2f7616fb41fece9b3d46fde2d546d70b46f9008e3.json new file mode 100644 index 0000000000..d60b424505 --- /dev/null +++ b/backend/.sqlx/query-2bf5f7f2cf9d85a5d23e5db2f7616fb41fece9b3d46fde2d546d70b46f9008e3.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO usage (id, is_workspace, month_, usage) \n VALUES ($1, FALSE, EXTRACT(YEAR FROM current_date) * 12 + EXTRACT(MONTH FROM current_date), $2) \n ON CONFLICT (id, is_workspace, month_) DO UPDATE SET usage = usage.usage + $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Int4" + ] + }, + "nullable": [] + }, + "hash": "2bf5f7f2cf9d85a5d23e5db2f7616fb41fece9b3d46fde2d546d70b46f9008e3" +} diff --git a/backend/.sqlx/query-2e9b3e718440f3c5269e9217a13076c565f3add98b6768b5476bd3afed11ea31.json b/backend/.sqlx/query-2e9b3e718440f3c5269e9217a13076c565f3add98b6768b5476bd3afed11ea31.json deleted file mode 100644 index 70c5674cdf..0000000000 --- a/backend/.sqlx/query-2e9b3e718440f3c5269e9217a13076c565f3add98b6768b5476bd3afed11ea31.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "INSERT INTO usage (id, is_workspace, month_, usage)\n VALUES ($1, FALSE, EXTRACT(YEAR FROM current_date) * 12 + EXTRACT(MONTH FROM current_date), 1)\n ON CONFLICT (id, is_workspace, month_) DO UPDATE SET usage = usage.usage + 1 \n RETURNING usage.usage", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "usage", - "type_info": "Int4" - } - ], - "parameters": { - "Left": [ - "Varchar" - ] - }, - "nullable": [ - false - ] - }, - "hash": "2e9b3e718440f3c5269e9217a13076c565f3add98b6768b5476bd3afed11ea31" -} diff --git a/backend/.sqlx/query-39401cb0db8d367b5beb2be0c13aa7595adae0eac4e4e3a888cb12b972d1a7ce.json b/backend/.sqlx/query-39401cb0db8d367b5beb2be0c13aa7595adae0eac4e4e3a888cb12b972d1a7ce.json new file mode 100644 index 0000000000..9906e0d56b --- /dev/null +++ b/backend/.sqlx/query-39401cb0db8d367b5beb2be0c13aa7595adae0eac4e4e3a888cb12b972d1a7ce.json @@ -0,0 +1,170 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT \n workspace_id, \n path, \n route_path, \n route_path_key,\n workspaced_route,\n script_path, \n summary,\n description,\n is_flow, \n http_method as \"http_method: _\", \n edited_by, \n email, \n edited_at, \n extra_perms, \n is_async, \n authentication_method as \"authentication_method: _\", \n static_asset_config as \"static_asset_config: _\", \n is_static_website,\n authentication_resource_path,\n wrap_body,\n raw_string\n FROM \n http_trigger\n WHERE \n workspace_id = $1 AND \n path = $2\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "workspace_id", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "path", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "route_path", + "type_info": "Varchar" + }, + { + "ordinal": 3, + "name": "route_path_key", + "type_info": "Varchar" + }, + { + "ordinal": 4, + "name": "workspaced_route", + "type_info": "Bool" + }, + { + "ordinal": 5, + "name": "script_path", + "type_info": "Varchar" + }, + { + "ordinal": 6, + "name": "summary", + "type_info": "Varchar" + }, + { + "ordinal": 7, + "name": "description", + "type_info": "Text" + }, + { + "ordinal": 8, + "name": "is_flow", + "type_info": "Bool" + }, + { + "ordinal": 9, + "name": "http_method: _", + "type_info": { + "Custom": { + "name": "http_method", + "kind": { + "Enum": [ + "get", + "post", + "put", + "delete", + "patch" + ] + } + } + } + }, + { + "ordinal": 10, + "name": "edited_by", + "type_info": "Varchar" + }, + { + "ordinal": 11, + "name": "email", + "type_info": "Varchar" + }, + { + "ordinal": 12, + "name": "edited_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 13, + "name": "extra_perms", + "type_info": "Jsonb" + }, + { + "ordinal": 14, + "name": "is_async", + "type_info": "Bool" + }, + { + "ordinal": 15, + "name": "authentication_method: _", + "type_info": { + "Custom": { + "name": "authentication_method", + "kind": { + "Enum": [ + "none", + "windmill", + "api_key", + "basic_http", + "custom_script", + "signature" + ] + } + } + } + }, + { + "ordinal": 16, + "name": "static_asset_config: _", + "type_info": "Jsonb" + }, + { + "ordinal": 17, + "name": "is_static_website", + "type_info": "Bool" + }, + { + "ordinal": 18, + "name": "authentication_resource_path", + "type_info": "Varchar" + }, + { + "ordinal": 19, + "name": "wrap_body", + "type_info": "Bool" + }, + { + "ordinal": 20, + "name": "raw_string", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false, + false, + false, + false, + false, + false, + true, + true, + false, + false, + false, + false, + false, + false, + false, + false, + true, + false, + true, + false, + false + ] + }, + "hash": "39401cb0db8d367b5beb2be0c13aa7595adae0eac4e4e3a888cb12b972d1a7ce" +} diff --git a/backend/.sqlx/query-3aaa6b6e362b10f2f3546b8595c60ab725295e0f20cb5f85d5fddc14d503a119.json b/backend/.sqlx/query-3aaa6b6e362b10f2f3546b8595c60ab725295e0f20cb5f85d5fddc14d503a119.json new file mode 100644 index 0000000000..5121dca128 --- /dev/null +++ b/backend/.sqlx/query-3aaa6b6e362b10f2f3546b8595c60ab725295e0f20cb5f85d5fddc14d503a119.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT \n path\n FROM\n script\n WHERE\n path ~ ANY($1) AND\n workspace_id = $2\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "path", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "TextArray", + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "3aaa6b6e362b10f2f3546b8595c60ab725295e0f20cb5f85d5fddc14d503a119" +} diff --git a/backend/.sqlx/query-faf2c77242e0ab39b33886edf3b742531bf1351d0be1c3631bde0adfe375497a.json b/backend/.sqlx/query-3e244a5057d4f1b4a18c0edac52cdf695c7e7aa0468d2686255de3d83719e6d0.json similarity index 53% rename from backend/.sqlx/query-faf2c77242e0ab39b33886edf3b742531bf1351d0be1c3631bde0adfe375497a.json rename to backend/.sqlx/query-3e244a5057d4f1b4a18c0edac52cdf695c7e7aa0468d2686255de3d83719e6d0.json index ea798bab76..e98bd34c2d 100644 --- a/backend/.sqlx/query-faf2c77242e0ab39b33886edf3b742531bf1351d0be1c3631bde0adfe375497a.json +++ b/backend/.sqlx/query-3e244a5057d4f1b4a18c0edac52cdf695c7e7aa0468d2686255de3d83719e6d0.json @@ -1,22 +1,22 @@ { "db_name": "PostgreSQL", - "query": "SELECT tag FROM v2_job WHERE id = $1", + "query": "SELECT path FROM script WHERE workspace_id = $1 AND archived = false", "describe": { "columns": [ { "ordinal": 0, - "name": "tag", + "name": "path", "type_info": "Varchar" } ], "parameters": { "Left": [ - "Uuid" + "Text" ] }, "nullable": [ false ] }, - "hash": "faf2c77242e0ab39b33886edf3b742531bf1351d0be1c3631bde0adfe375497a" + "hash": "3e244a5057d4f1b4a18c0edac52cdf695c7e7aa0468d2686255de3d83719e6d0" } diff --git a/backend/.sqlx/query-3f05e6186050a7ce6d8efb41067d3c5282319fe7e041f114e02fb22b91716637.json b/backend/.sqlx/query-3f05e6186050a7ce6d8efb41067d3c5282319fe7e041f114e02fb22b91716637.json new file mode 100644 index 0000000000..86d22794c6 --- /dev/null +++ b/backend/.sqlx/query-3f05e6186050a7ce6d8efb41067d3c5282319fe7e041f114e02fb22b91716637.json @@ -0,0 +1,60 @@ +{ + "db_name": "PostgreSQL", + "query": "\n UPDATE \n http_trigger \n SET \n route_path = $1, \n route_path_key = $2, \n workspaced_route = $3,\n wrap_body = $4,\n raw_string = $5,\n authentication_resource_path = $6,\n script_path = $7, \n path = $8, \n is_flow = $9, \n http_method = $10, \n static_asset_config = $11, \n edited_by = $12, \n email = $13, \n is_async = $14, \n authentication_method = $15, \n summary = $16,\n description = $17,\n edited_at = now(), \n is_static_website = $18\n WHERE \n workspace_id = $19 AND \n path = $20\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Bool", + "Bool", + "Bool", + "Varchar", + "Varchar", + "Varchar", + "Bool", + { + "Custom": { + "name": "http_method", + "kind": { + "Enum": [ + "get", + "post", + "put", + "delete", + "patch" + ] + } + } + }, + "Jsonb", + "Varchar", + "Varchar", + "Bool", + { + "Custom": { + "name": "authentication_method", + "kind": { + "Enum": [ + "none", + "windmill", + "api_key", + "basic_http", + "custom_script", + "signature" + ] + } + } + }, + "Varchar", + "Text", + "Bool", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "3f05e6186050a7ce6d8efb41067d3c5282319fe7e041f114e02fb22b91716637" +} diff --git a/backend/.sqlx/query-4228b098883408323bd8413ee094454b95962047458a6927d19ac0d3e7b3f0fa.json b/backend/.sqlx/query-4228b098883408323bd8413ee094454b95962047458a6927d19ac0d3e7b3f0fa.json new file mode 100644 index 0000000000..365bce7366 --- /dev/null +++ b/backend/.sqlx/query-4228b098883408323bd8413ee094454b95962047458a6927d19ac0d3e7b3f0fa.json @@ -0,0 +1,169 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT \n workspace_id, \n workspaced_route,\n path, \n route_path, \n route_path_key, \n authentication_resource_path,\n script_path, \n is_flow, \n summary,\n description,\n edited_by, \n edited_at, \n email, \n extra_perms, \n is_async, \n authentication_method AS \"authentication_method: _\", \n http_method AS \"http_method: _\", \n static_asset_config AS \"static_asset_config: _\", \n is_static_website,\n wrap_body,\n raw_string\n FROM http_trigger\n WHERE workspace_id = $1\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "workspace_id", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "workspaced_route", + "type_info": "Bool" + }, + { + "ordinal": 2, + "name": "path", + "type_info": "Varchar" + }, + { + "ordinal": 3, + "name": "route_path", + "type_info": "Varchar" + }, + { + "ordinal": 4, + "name": "route_path_key", + "type_info": "Varchar" + }, + { + "ordinal": 5, + "name": "authentication_resource_path", + "type_info": "Varchar" + }, + { + "ordinal": 6, + "name": "script_path", + "type_info": "Varchar" + }, + { + "ordinal": 7, + "name": "is_flow", + "type_info": "Bool" + }, + { + "ordinal": 8, + "name": "summary", + "type_info": "Varchar" + }, + { + "ordinal": 9, + "name": "description", + "type_info": "Text" + }, + { + "ordinal": 10, + "name": "edited_by", + "type_info": "Varchar" + }, + { + "ordinal": 11, + "name": "edited_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 12, + "name": "email", + "type_info": "Varchar" + }, + { + "ordinal": 13, + "name": "extra_perms", + "type_info": "Jsonb" + }, + { + "ordinal": 14, + "name": "is_async", + "type_info": "Bool" + }, + { + "ordinal": 15, + "name": "authentication_method: _", + "type_info": { + "Custom": { + "name": "authentication_method", + "kind": { + "Enum": [ + "none", + "windmill", + "api_key", + "basic_http", + "custom_script", + "signature" + ] + } + } + } + }, + { + "ordinal": 16, + "name": "http_method: _", + "type_info": { + "Custom": { + "name": "http_method", + "kind": { + "Enum": [ + "get", + "post", + "put", + "delete", + "patch" + ] + } + } + } + }, + { + "ordinal": 17, + "name": "static_asset_config: _", + "type_info": "Jsonb" + }, + { + "ordinal": 18, + "name": "is_static_website", + "type_info": "Bool" + }, + { + "ordinal": 19, + "name": "wrap_body", + "type_info": "Bool" + }, + { + "ordinal": 20, + "name": "raw_string", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false, + false, + false, + false, + false, + true, + false, + false, + true, + true, + false, + false, + false, + false, + false, + false, + false, + true, + false, + false, + false + ] + }, + "hash": "4228b098883408323bd8413ee094454b95962047458a6927d19ac0d3e7b3f0fa" +} diff --git a/backend/.sqlx/query-488dd591096b2b47787afdc3a1d73917ed13269f2ee20b86df79fca2c8efe672.json b/backend/.sqlx/query-488dd591096b2b47787afdc3a1d73917ed13269f2ee20b86df79fca2c8efe672.json index da5eb1ceea..561e202d1d 100644 --- a/backend/.sqlx/query-488dd591096b2b47787afdc3a1d73917ed13269f2ee20b86df79fca2c8efe672.json +++ b/backend/.sqlx/query-488dd591096b2b47787afdc3a1d73917ed13269f2ee20b86df79fca2c8efe672.json @@ -70,7 +70,8 @@ "csharp", "oracledb", "nu", - "java" + "java", + "duckdb" ] } } diff --git a/backend/.sqlx/query-4aaab98ebdaa90f1edf49ac96fba6c391c4d0054a618b861464ee37239f1f1e0.json b/backend/.sqlx/query-4aaab98ebdaa90f1edf49ac96fba6c391c4d0054a618b861464ee37239f1f1e0.json index d8a93c797d..65fa6cc9eb 100644 --- a/backend/.sqlx/query-4aaab98ebdaa90f1edf49ac96fba6c391c4d0054a618b861464ee37239f1f1e0.json +++ b/backend/.sqlx/query-4aaab98ebdaa90f1edf49ac96fba6c391c4d0054a618b861464ee37239f1f1e0.json @@ -137,7 +137,8 @@ "csharp", "oracledb", "nu", - "java" + "java", + "duckdb" ] } } diff --git a/backend/.sqlx/query-4dc21dda58758a84bbc1b7f9328efb9806223d1684c53907ff825bc0228ec18d.json b/backend/.sqlx/query-4dc21dda58758a84bbc1b7f9328efb9806223d1684c53907ff825bc0228ec18d.json new file mode 100644 index 0000000000..e0c5d2bb4c --- /dev/null +++ b/backend/.sqlx/query-4dc21dda58758a84bbc1b7f9328efb9806223d1684c53907ff825bc0228ec18d.json @@ -0,0 +1,35 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT \n path,\n summary,\n description\n FROM\n flow\n WHERE\n path ~ ANY($1) AND\n workspace_id = $2\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "path", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "summary", + "type_info": "Text" + }, + { + "ordinal": 2, + "name": "description", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "TextArray", + "Text" + ] + }, + "nullable": [ + false, + false, + false + ] + }, + "hash": "4dc21dda58758a84bbc1b7f9328efb9806223d1684c53907ff825bc0228ec18d" +} diff --git a/backend/.sqlx/query-4e5273b9ce05f6ee2dfd5f14c8574a0cf43682480452f7dbe23012320fe7fe25.json b/backend/.sqlx/query-4e5273b9ce05f6ee2dfd5f14c8574a0cf43682480452f7dbe23012320fe7fe25.json new file mode 100644 index 0000000000..8176539a30 --- /dev/null +++ b/backend/.sqlx/query-4e5273b9ce05f6ee2dfd5f14c8574a0cf43682480452f7dbe23012320fe7fe25.json @@ -0,0 +1,58 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT \n tag, \n script_lang AS \"script_lang!: _\"\n FROM \n v2_job\n WHERE \n id = $1\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "tag", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "script_lang!: _", + "type_info": { + "Custom": { + "name": "script_lang", + "kind": { + "Enum": [ + "python3", + "deno", + "go", + "bash", + "postgresql", + "nativets", + "bun", + "mysql", + "bigquery", + "snowflake", + "graphql", + "powershell", + "mssql", + "php", + "bunnative", + "rust", + "ansible", + "csharp", + "oracledb", + "nu", + "java", + "duckdb" + ] + } + } + } + } + ], + "parameters": { + "Left": [ + "Uuid" + ] + }, + "nullable": [ + false, + true + ] + }, + "hash": "4e5273b9ce05f6ee2dfd5f14c8574a0cf43682480452f7dbe23012320fe7fe25" +} diff --git a/backend/.sqlx/query-3895cee539a24b4c6ea89fa7a835fc62bc93b0530efba09fc3c32a8f93eaabb1.json b/backend/.sqlx/query-5430f7728c1e9b539cc8aad29ca9e6733943278998d3df62a9486607827e59ec.json similarity index 68% rename from backend/.sqlx/query-3895cee539a24b4c6ea89fa7a835fc62bc93b0530efba09fc3c32a8f93eaabb1.json rename to backend/.sqlx/query-5430f7728c1e9b539cc8aad29ca9e6733943278998d3df62a9486607827e59ec.json index 66431239db..f3961847f4 100644 --- a/backend/.sqlx/query-3895cee539a24b4c6ea89fa7a835fc62bc93b0530efba09fc3c32a8f93eaabb1.json +++ b/backend/.sqlx/query-5430f7728c1e9b539cc8aad29ca9e6733943278998d3df62a9486607827e59ec.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "WITH active_users AS (SELECT distinct username as email FROM audit WHERE timestamp > NOW() - INTERVAL '1 month' AND (operation = 'users.login' OR operation = 'oauth.login')),\n authors as (SELECT distinct email FROM usr WHERE usr.operator IS false)\n SELECT email, email NOT IN (SELECT email FROM authors) as operator_only, login_type::text, verified, super_admin, devops, name, company, username\n FROM password\n WHERE email IN (SELECT email FROM active_users)\n ORDER BY super_admin DESC, devops DESC\n LIMIT $1 OFFSET $2", + "query": "WITH active_users AS (SELECT distinct username as email FROM audit WHERE timestamp > NOW() - INTERVAL '1 month' AND (operation = 'users.login' OR operation = 'oauth.login' OR operation = 'users.token.refresh')),\n authors as (SELECT distinct email FROM usr WHERE usr.operator IS false)\n SELECT email, email NOT IN (SELECT email FROM authors) as operator_only, login_type::text, verified, super_admin, devops, name, company, username\n FROM password\n WHERE email IN (SELECT email FROM active_users)\n ORDER BY super_admin DESC, devops DESC\n LIMIT $1 OFFSET $2", "describe": { "columns": [ { @@ -67,5 +67,5 @@ true ] }, - "hash": "3895cee539a24b4c6ea89fa7a835fc62bc93b0530efba09fc3c32a8f93eaabb1" + "hash": "5430f7728c1e9b539cc8aad29ca9e6733943278998d3df62a9486607827e59ec" } diff --git a/backend/.sqlx/query-553108ba3c0b8d579800bc8b5a4f887d79fb4c13b60b19c4913a8db18521958c.json b/backend/.sqlx/query-553108ba3c0b8d579800bc8b5a4f887d79fb4c13b60b19c4913a8db18521958c.json deleted file mode 100644 index e3d2e205f1..0000000000 --- a/backend/.sqlx/query-553108ba3c0b8d579800bc8b5a4f887d79fb4c13b60b19c4913a8db18521958c.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE v2_job_queue q SET suspend = 0\n FROM v2_job j, v2_job_status f\n WHERE parent_job = $1\n AND f.id = j.id AND q.id = j.id\n AND suspend = $2 AND (f.flow_status->'step')::int = 0", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Uuid", - "Int4" - ] - }, - "nullable": [] - }, - "hash": "553108ba3c0b8d579800bc8b5a4f887d79fb4c13b60b19c4913a8db18521958c" -} diff --git a/backend/.sqlx/query-56b2326015fde12b1a4efa226518566101dd27a0f3363884781071d417f8b7e7.json b/backend/.sqlx/query-56b2326015fde12b1a4efa226518566101dd27a0f3363884781071d417f8b7e7.json new file mode 100644 index 0000000000..9da0554266 --- /dev/null +++ b/backend/.sqlx/query-56b2326015fde12b1a4efa226518566101dd27a0f3363884781071d417f8b7e7.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO usage (id, is_workspace, month_, usage)\n VALUES ($1, FALSE, EXTRACT(YEAR FROM current_date) * 12 + EXTRACT(MONTH FROM current_date), 1)\n ON CONFLICT (id, is_workspace, month_) DO UPDATE SET usage = usage.usage + 1 \n RETURNING usage.usage", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "usage", + "type_info": "Int4" + } + ], + "parameters": { + "Left": [ + "Varchar" + ] + }, + "nullable": [ + false + ] + }, + "hash": "56b2326015fde12b1a4efa226518566101dd27a0f3363884781071d417f8b7e7" +} diff --git a/backend/.sqlx/query-61c29d684e8e683e839a6d7210b3b9b96854e5bfd752e45922c358c42ebea0c4.json b/backend/.sqlx/query-61c29d684e8e683e839a6d7210b3b9b96854e5bfd752e45922c358c42ebea0c4.json index be472306a0..1a30ad011f 100644 --- a/backend/.sqlx/query-61c29d684e8e683e839a6d7210b3b9b96854e5bfd752e45922c358c42ebea0c4.json +++ b/backend/.sqlx/query-61c29d684e8e683e839a6d7210b3b9b96854e5bfd752e45922c358c42ebea0c4.json @@ -61,7 +61,8 @@ "csharp", "oracledb", "nu", - "java" + "java", + "duckdb" ] } } diff --git a/backend/.sqlx/query-621e9a2a53187dac3ebed62f0d645b692815f1594bf302dbebd5f80d5d22b98e.json b/backend/.sqlx/query-621e9a2a53187dac3ebed62f0d645b692815f1594bf302dbebd5f80d5d22b98e.json deleted file mode 100644 index 6a66bb6f22..0000000000 --- a/backend/.sqlx/query-621e9a2a53187dac3ebed62f0d645b692815f1594bf302dbebd5f80d5d22b98e.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "INSERT INTO usage (id, is_workspace, month_, usage)\n VALUES ($1, TRUE, EXTRACT(YEAR FROM current_date) * 12 + EXTRACT(MONTH FROM current_date), 1)\n ON CONFLICT (id, is_workspace, month_) DO UPDATE SET usage = usage.usage + 1 \n RETURNING usage.usage", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "usage", - "type_info": "Int4" - } - ], - "parameters": { - "Left": [ - "Varchar" - ] - }, - "nullable": [ - false - ] - }, - "hash": "621e9a2a53187dac3ebed62f0d645b692815f1594bf302dbebd5f80d5d22b98e" -} diff --git a/backend/.sqlx/query-6b6aca712df657f81e74a438f6753bc72e787a3f768040c8c73ea86313badb71.json b/backend/.sqlx/query-6b6aca712df657f81e74a438f6753bc72e787a3f768040c8c73ea86313badb71.json new file mode 100644 index 0000000000..fe40dfd0cd --- /dev/null +++ b/backend/.sqlx/query-6b6aca712df657f81e74a438f6753bc72e787a3f768040c8c73ea86313badb71.json @@ -0,0 +1,55 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT\n route_path,\n http_method AS \"http_method: _\",\n is_async,\n workspaced_route\n FROM\n http_trigger\n WHERE\n path ~ ANY($1) AND\n route_path ~ ANY($2) AND\n workspace_id = $3\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "route_path", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "http_method: _", + "type_info": { + "Custom": { + "name": "http_method", + "kind": { + "Enum": [ + "get", + "post", + "put", + "delete", + "patch" + ] + } + } + } + }, + { + "ordinal": 2, + "name": "is_async", + "type_info": "Bool" + }, + { + "ordinal": 3, + "name": "workspaced_route", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "TextArray", + "TextArray", + "Text" + ] + }, + "nullable": [ + false, + false, + false, + false + ] + }, + "hash": "6b6aca712df657f81e74a438f6753bc72e787a3f768040c8c73ea86313badb71" +} diff --git a/backend/.sqlx/query-6e2564dc37ee967c634deade67ceabc4c516418e595dee9cd7d651acc1f4af6e.json b/backend/.sqlx/query-6e2564dc37ee967c634deade67ceabc4c516418e595dee9cd7d651acc1f4af6e.json index 1ab5672e7d..39ebee28b4 100644 --- a/backend/.sqlx/query-6e2564dc37ee967c634deade67ceabc4c516418e595dee9cd7d651acc1f4af6e.json +++ b/backend/.sqlx/query-6e2564dc37ee967c634deade67ceabc4c516418e595dee9cd7d651acc1f4af6e.json @@ -147,7 +147,8 @@ "csharp", "oracledb", "nu", - "java" + "java", + "duckdb" ] } } diff --git a/backend/.sqlx/query-714fb0f66ceb536aee8cb9ae0144757b999d25870fda37fe904e09dd5c742015.json b/backend/.sqlx/query-714fb0f66ceb536aee8cb9ae0144757b999d25870fda37fe904e09dd5c742015.json new file mode 100644 index 0000000000..9f3f4b0c8c --- /dev/null +++ b/backend/.sqlx/query-714fb0f66ceb536aee8cb9ae0144757b999d25870fda37fe904e09dd5c742015.json @@ -0,0 +1,93 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT\n route_path,\n http_method AS \"http_method: _\",\n is_async,\n workspaced_route,\n summary,\n description,\n authentication_method AS \"authentication_method: _\",\n authentication_resource_path\n FROM\n http_trigger\n WHERE\n path ~ ANY($1) AND\n route_path ~ ANY($2) AND\n workspace_id = $3\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "route_path", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "http_method: _", + "type_info": { + "Custom": { + "name": "http_method", + "kind": { + "Enum": [ + "get", + "post", + "put", + "delete", + "patch" + ] + } + } + } + }, + { + "ordinal": 2, + "name": "is_async", + "type_info": "Bool" + }, + { + "ordinal": 3, + "name": "workspaced_route", + "type_info": "Bool" + }, + { + "ordinal": 4, + "name": "summary", + "type_info": "Varchar" + }, + { + "ordinal": 5, + "name": "description", + "type_info": "Text" + }, + { + "ordinal": 6, + "name": "authentication_method: _", + "type_info": { + "Custom": { + "name": "authentication_method", + "kind": { + "Enum": [ + "none", + "windmill", + "api_key", + "basic_http", + "custom_script", + "signature" + ] + } + } + } + }, + { + "ordinal": 7, + "name": "authentication_resource_path", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "TextArray", + "TextArray", + "Text" + ] + }, + "nullable": [ + false, + false, + false, + false, + true, + true, + false, + true + ] + }, + "hash": "714fb0f66ceb536aee8cb9ae0144757b999d25870fda37fe904e09dd5c742015" +} diff --git a/backend/.sqlx/query-804fc11e35f4afc0db194b6fe2594f91df7e588d4d2431bc85f4d8734920c8bf.json b/backend/.sqlx/query-804fc11e35f4afc0db194b6fe2594f91df7e588d4d2431bc85f4d8734920c8bf.json index 89745202b0..7933311a4c 100644 --- a/backend/.sqlx/query-804fc11e35f4afc0db194b6fe2594f91df7e588d4d2431bc85f4d8734920c8bf.json +++ b/backend/.sqlx/query-804fc11e35f4afc0db194b6fe2594f91df7e588d4d2431bc85f4d8734920c8bf.json @@ -34,7 +34,8 @@ "csharp", "oracledb", "nu", - "java" + "java", + "duckdb" ] } } diff --git a/backend/.sqlx/query-8373b2649ab46310860adbdd7b717261771ac61d46d82d42d085ffebeb18be06.json b/backend/.sqlx/query-8373b2649ab46310860adbdd7b717261771ac61d46d82d42d085ffebeb18be06.json new file mode 100644 index 0000000000..3df7f08318 --- /dev/null +++ b/backend/.sqlx/query-8373b2649ab46310860adbdd7b717261771ac61d46d82d42d085ffebeb18be06.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT distinct(path) FROM script WHERE workspace_id = $1 AND archived = true", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "path", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "8373b2649ab46310860adbdd7b717261771ac61d46d82d42d085ffebeb18be06" +} diff --git a/backend/.sqlx/query-83f64dd93b1ddc03b84681d65d9be69959987cbac1d83b64225fd1bf9ab047c9.json b/backend/.sqlx/query-83f64dd93b1ddc03b84681d65d9be69959987cbac1d83b64225fd1bf9ab047c9.json new file mode 100644 index 0000000000..3ca69e27dd --- /dev/null +++ b/backend/.sqlx/query-83f64dd93b1ddc03b84681d65d9be69959987cbac1d83b64225fd1bf9ab047c9.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO usage (id, is_workspace, month_, usage)\n VALUES ($1, TRUE, EXTRACT(YEAR FROM current_date) * 12 + EXTRACT(MONTH FROM current_date), 1)\n ON CONFLICT (id, is_workspace, month_) DO UPDATE SET usage = usage.usage + 1 \n RETURNING usage.usage", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "usage", + "type_info": "Int4" + } + ], + "parameters": { + "Left": [ + "Varchar" + ] + }, + "nullable": [ + false + ] + }, + "hash": "83f64dd93b1ddc03b84681d65d9be69959987cbac1d83b64225fd1bf9ab047c9" +} diff --git a/backend/.sqlx/query-997586ac14384db2c0eeee1bb3382cc6ae013695d0cda6da9ab848ca1b9a9606.json b/backend/.sqlx/query-997586ac14384db2c0eeee1bb3382cc6ae013695d0cda6da9ab848ca1b9a9606.json new file mode 100644 index 0000000000..1edf26100f --- /dev/null +++ b/backend/.sqlx/query-997586ac14384db2c0eeee1bb3382cc6ae013695d0cda6da9ab848ca1b9a9606.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE worker_ping SET\nping_at = now(),\njobs_executed = 1,\ncurrent_job_id = $1,\ncurrent_job_workspace_id = 'admins'\nWHERE worker = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Uuid", + "Text" + ] + }, + "nullable": [] + }, + "hash": "997586ac14384db2c0eeee1bb3382cc6ae013695d0cda6da9ab848ca1b9a9606" +} diff --git a/backend/.sqlx/query-a8bcae108af1eda6efe3a4b8c6f8807bc464a81c0883e68f5a69b89b94b0b34b.json b/backend/.sqlx/query-a8bcae108af1eda6efe3a4b8c6f8807bc464a81c0883e68f5a69b89b94b0b34b.json index a2a19a6c35..a3cb5a82b9 100644 --- a/backend/.sqlx/query-a8bcae108af1eda6efe3a4b8c6f8807bc464a81c0883e68f5a69b89b94b0b34b.json +++ b/backend/.sqlx/query-a8bcae108af1eda6efe3a4b8c6f8807bc464a81c0883e68f5a69b89b94b0b34b.json @@ -36,7 +36,8 @@ "csharp", "oracledb", "nu", - "java" + "java", + "duckdb" ] } } diff --git a/backend/.sqlx/query-a9db7b2f435bb82acb8c5eeb7f800b28f3256491fdaa168591adc7b4b9f3327a.json b/backend/.sqlx/query-a9db7b2f435bb82acb8c5eeb7f800b28f3256491fdaa168591adc7b4b9f3327a.json index 8cc09349ec..16fea8720d 100644 --- a/backend/.sqlx/query-a9db7b2f435bb82acb8c5eeb7f800b28f3256491fdaa168591adc7b4b9f3327a.json +++ b/backend/.sqlx/query-a9db7b2f435bb82acb8c5eeb7f800b28f3256491fdaa168591adc7b4b9f3327a.json @@ -41,7 +41,8 @@ "csharp", "oracledb", "nu", - "java" + "java", + "duckdb" ] } } diff --git a/backend/.sqlx/query-ab04cda71f8e2be9acbecabe1ee5ef756b8e5c1955fbe111df9ee171dc262338.json b/backend/.sqlx/query-ab04cda71f8e2be9acbecabe1ee5ef756b8e5c1955fbe111df9ee171dc262338.json index 47e4f47005..c8cbb8e10d 100644 --- a/backend/.sqlx/query-ab04cda71f8e2be9acbecabe1ee5ef756b8e5c1955fbe111df9ee171dc262338.json +++ b/backend/.sqlx/query-ab04cda71f8e2be9acbecabe1ee5ef756b8e5c1955fbe111df9ee171dc262338.json @@ -65,7 +65,8 @@ "csharp", "oracledb", "nu", - "java" + "java", + "duckdb" ] } } diff --git a/backend/.sqlx/query-c3025cdb6e421e1225d420e8b1efd18d1dd3bb2fac53c1f2df648b61fb7488aa.json b/backend/.sqlx/query-c3025cdb6e421e1225d420e8b1efd18d1dd3bb2fac53c1f2df648b61fb7488aa.json deleted file mode 100644 index 638417aabb..0000000000 --- a/backend/.sqlx/query-c3025cdb6e421e1225d420e8b1efd18d1dd3bb2fac53c1f2df648b61fb7488aa.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE worker_ping SET \nping_at = now(), \njobs_executed = 1, \ncurrent_job_id = $1, \ncurrent_job_workspace_id = 'admins' \nWHERE worker = $2", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Uuid", - "Text" - ] - }, - "nullable": [] - }, - "hash": "c3025cdb6e421e1225d420e8b1efd18d1dd3bb2fac53c1f2df648b61fb7488aa" -} diff --git a/backend/.sqlx/query-ca3ba808e020c8c7a35eaef842b20cfeee64fd47ded72fce55cc75e0bbb291a8.json b/backend/.sqlx/query-ca3ba808e020c8c7a35eaef842b20cfeee64fd47ded72fce55cc75e0bbb291a8.json new file mode 100644 index 0000000000..6d44fb2840 --- /dev/null +++ b/backend/.sqlx/query-ca3ba808e020c8c7a35eaef842b20cfeee64fd47ded72fce55cc75e0bbb291a8.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO usage (id, is_workspace, month_, usage) \n VALUES ($1, TRUE, EXTRACT(YEAR FROM current_date) * 12 + EXTRACT(MONTH FROM current_date), $2) \n ON CONFLICT (id, is_workspace, month_) DO UPDATE SET usage = usage.usage + $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Int4" + ] + }, + "nullable": [] + }, + "hash": "ca3ba808e020c8c7a35eaef842b20cfeee64fd47ded72fce55cc75e0bbb291a8" +} diff --git a/backend/.sqlx/query-cb3862634f18160207ee2621ddfca43f00456a27fda32583846497116f92f96c.json b/backend/.sqlx/query-cb3862634f18160207ee2621ddfca43f00456a27fda32583846497116f92f96c.json new file mode 100644 index 0000000000..8ccf623719 --- /dev/null +++ b/backend/.sqlx/query-cb3862634f18160207ee2621ddfca43f00456a27fda32583846497116f92f96c.json @@ -0,0 +1,38 @@ +{ + "db_name": "PostgreSQL", + "query": "WITH active_users as (SELECT distinct username as email FROM audit WHERE timestamp > NOW() - INTERVAL '1 month' AND (operation = 'users.login' OR operation = 'oauth.login' OR operation = 'users.token.refresh')),\n active_authors as (SELECT distinct email FROM usr WHERE usr.operator IS false AND email IN (SELECT email FROM active_users)),\n active_authors_agg as (SELECT array_agg(email) as authors FROM active_authors),\n active_ops_agg as (SELECT array_agg(email) as operators from active_users WHERE email NOT IN (SELECT email FROM active_authors))\n SELECT active_authors_agg.authors, active_ops_agg.operators, array_length(active_authors_agg.authors, 1) as author_count, array_length(active_ops_agg.operators, 1) as operator_count FROM active_authors_agg, active_ops_agg", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "authors", + "type_info": "VarcharArray" + }, + { + "ordinal": 1, + "name": "operators", + "type_info": "VarcharArray" + }, + { + "ordinal": 2, + "name": "author_count", + "type_info": "Int4" + }, + { + "ordinal": 3, + "name": "operator_count", + "type_info": "Int4" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + null, + null, + null, + null + ] + }, + "hash": "cb3862634f18160207ee2621ddfca43f00456a27fda32583846497116f92f96c" +} diff --git a/backend/.sqlx/query-cccdcb7fe7968eadfc04d8957a8e98b2f2d92a6d7f687a9dd5a70edb3d5a63e6.json b/backend/.sqlx/query-cccdcb7fe7968eadfc04d8957a8e98b2f2d92a6d7f687a9dd5a70edb3d5a63e6.json index 412b64c480..94f591b3da 100644 --- a/backend/.sqlx/query-cccdcb7fe7968eadfc04d8957a8e98b2f2d92a6d7f687a9dd5a70edb3d5a63e6.json +++ b/backend/.sqlx/query-cccdcb7fe7968eadfc04d8957a8e98b2f2d92a6d7f687a9dd5a70edb3d5a63e6.json @@ -70,7 +70,8 @@ "csharp", "oracledb", "nu", - "java" + "java", + "duckdb" ] } } diff --git a/backend/.sqlx/query-cce991f582bc9d2ba28a5b2b41c679366bb07bc6a100727721a787160ac6910c.json b/backend/.sqlx/query-cce991f582bc9d2ba28a5b2b41c679366bb07bc6a100727721a787160ac6910c.json deleted file mode 100644 index f5d905a157..0000000000 --- a/backend/.sqlx/query-cce991f582bc9d2ba28a5b2b41c679366bb07bc6a100727721a787160ac6910c.json +++ /dev/null @@ -1,38 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "WITH active_users as (SELECT distinct username as email FROM audit WHERE timestamp > NOW() - INTERVAL '1 month' AND (operation = 'users.login' OR operation = 'oauth.login')),\n active_authors as (SELECT distinct email FROM usr WHERE usr.operator IS false AND email IN (SELECT email FROM active_users)),\n active_authors_agg as (SELECT array_agg(email) as authors FROM active_authors),\n active_ops_agg as (SELECT array_agg(email) as operators from active_users WHERE email NOT IN (SELECT email FROM active_authors))\n SELECT active_authors_agg.authors, active_ops_agg.operators, array_length(active_authors_agg.authors, 1) as author_count, array_length(active_ops_agg.operators, 1) as operator_count FROM active_authors_agg, active_ops_agg", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "authors", - "type_info": "VarcharArray" - }, - { - "ordinal": 1, - "name": "operators", - "type_info": "VarcharArray" - }, - { - "ordinal": 2, - "name": "author_count", - "type_info": "Int4" - }, - { - "ordinal": 3, - "name": "operator_count", - "type_info": "Int4" - } - ], - "parameters": { - "Left": [] - }, - "nullable": [ - null, - null, - null, - null - ] - }, - "hash": "cce991f582bc9d2ba28a5b2b41c679366bb07bc6a100727721a787160ac6910c" -} diff --git a/backend/.sqlx/query-d15f02f090b8d1a7e816fe11b2e0867540ab6bb02ac6bf82decc220dce0ab048.json b/backend/.sqlx/query-d15f02f090b8d1a7e816fe11b2e0867540ab6bb02ac6bf82decc220dce0ab048.json index 0a2b8c3a0c..3a23232dcc 100644 --- a/backend/.sqlx/query-d15f02f090b8d1a7e816fe11b2e0867540ab6bb02ac6bf82decc220dce0ab048.json +++ b/backend/.sqlx/query-d15f02f090b8d1a7e816fe11b2e0867540ab6bb02ac6bf82decc220dce0ab048.json @@ -42,7 +42,8 @@ "csharp", "oracledb", "nu", - "java" + "java", + "duckdb" ] } } diff --git a/backend/.sqlx/query-d41df114e78873111e409e1182bccc0024acf17ce740f70d2243e5e2a2b25163.json b/backend/.sqlx/query-d41df114e78873111e409e1182bccc0024acf17ce740f70d2243e5e2a2b25163.json new file mode 100644 index 0000000000..febfc4c21f --- /dev/null +++ b/backend/.sqlx/query-d41df114e78873111e409e1182bccc0024acf17ce740f70d2243e5e2a2b25163.json @@ -0,0 +1,35 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT \n path,\n summary,\n description\n FROM\n script\n WHERE\n path ~ ANY($1) AND\n workspace_id = $2\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "path", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "summary", + "type_info": "Text" + }, + { + "ordinal": 2, + "name": "description", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "TextArray", + "Text" + ] + }, + "nullable": [ + false, + false, + false + ] + }, + "hash": "d41df114e78873111e409e1182bccc0024acf17ce740f70d2243e5e2a2b25163" +} diff --git a/backend/.sqlx/query-e38240e6d50bfe60e1c2b649588eb41dcef121ed161db04b2568ac2d990aed7c.json b/backend/.sqlx/query-e38240e6d50bfe60e1c2b649588eb41dcef121ed161db04b2568ac2d990aed7c.json deleted file mode 100644 index 1ff35d7227..0000000000 --- a/backend/.sqlx/query-e38240e6d50bfe60e1c2b649588eb41dcef121ed161db04b2568ac2d990aed7c.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "INSERT INTO usage (id, is_workspace, month_, usage) \n VALUES ($1, FALSE, EXTRACT(YEAR FROM current_date) * 12 + EXTRACT(MONTH FROM current_date), $2) \n ON CONFLICT (id, is_workspace, month_) DO UPDATE SET usage = usage.usage + $2", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Varchar", - "Int4" - ] - }, - "nullable": [] - }, - "hash": "e38240e6d50bfe60e1c2b649588eb41dcef121ed161db04b2568ac2d990aed7c" -} diff --git a/backend/.sqlx/query-ed99d4d088d0fd0c01f29803b12e99ae0a53d0b1feaa67737da409c51c1b6751.json b/backend/.sqlx/query-ed99d4d088d0fd0c01f29803b12e99ae0a53d0b1feaa67737da409c51c1b6751.json new file mode 100644 index 0000000000..a7ed7469d9 --- /dev/null +++ b/backend/.sqlx/query-ed99d4d088d0fd0c01f29803b12e99ae0a53d0b1feaa67737da409c51c1b6751.json @@ -0,0 +1,59 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO http_trigger (\n workspace_id, \n path, \n route_path, \n route_path_key,\n workspaced_route,\n authentication_resource_path,\n wrap_body,\n raw_string,\n script_path, \n summary,\n description,\n is_flow, \n is_async, \n authentication_method, \n http_method, \n static_asset_config, \n edited_by, \n email, \n edited_at, \n is_static_website\n ) \n VALUES (\n $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, now(), $19\n )\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Varchar", + "Varchar", + "Bool", + "Varchar", + "Bool", + "Bool", + "Varchar", + "Varchar", + "Text", + "Bool", + "Bool", + { + "Custom": { + "name": "authentication_method", + "kind": { + "Enum": [ + "none", + "windmill", + "api_key", + "basic_http", + "custom_script", + "signature" + ] + } + } + }, + { + "Custom": { + "name": "http_method", + "kind": { + "Enum": [ + "get", + "post", + "put", + "delete", + "patch" + ] + } + } + }, + "Jsonb", + "Varchar", + "Varchar", + "Bool" + ] + }, + "nullable": [] + }, + "hash": "ed99d4d088d0fd0c01f29803b12e99ae0a53d0b1feaa67737da409c51c1b6751" +} diff --git a/backend/.sqlx/query-f1dbcb6e6d82d17c19eb88c0e67dc1cb8baf5bd40b75a2a9cd3ebac440fda632.json b/backend/.sqlx/query-f1dbcb6e6d82d17c19eb88c0e67dc1cb8baf5bd40b75a2a9cd3ebac440fda632.json new file mode 100644 index 0000000000..b58fad894b --- /dev/null +++ b/backend/.sqlx/query-f1dbcb6e6d82d17c19eb88c0e67dc1cb8baf5bd40b75a2a9cd3ebac440fda632.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE v2_job_queue q SET suspend = 0\n FROM v2_job j, v2_job_status f\n WHERE q.workspace_id = $1 AND q.suspend = $3 AND j.parent_job = $2\n AND f.id = j.id AND q.id = j.id\n AND (f.flow_status->'step')::int = 0", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Uuid", + "Int4" + ] + }, + "nullable": [] + }, + "hash": "f1dbcb6e6d82d17c19eb88c0e67dc1cb8baf5bd40b75a2a9cd3ebac440fda632" +} diff --git a/backend/.sqlx/query-f22964772dc2d67aee437bbbd08b64792c00da1d713d7ca8f9904ccce7bfdae7.json b/backend/.sqlx/query-f22964772dc2d67aee437bbbd08b64792c00da1d713d7ca8f9904ccce7bfdae7.json index b3a352de55..b400be3485 100644 --- a/backend/.sqlx/query-f22964772dc2d67aee437bbbd08b64792c00da1d713d7ca8f9904ccce7bfdae7.json +++ b/backend/.sqlx/query-f22964772dc2d67aee437bbbd08b64792c00da1d713d7ca8f9904ccce7bfdae7.json @@ -147,7 +147,8 @@ "csharp", "oracledb", "nu", - "java" + "java", + "duckdb" ] } } diff --git a/backend/.sqlx/query-ff0403790674cdb07022af71c2377afbd8b3a660b3be27514b517c077c63c238.json b/backend/.sqlx/query-ff0403790674cdb07022af71c2377afbd8b3a660b3be27514b517c077c63c238.json index c7644fa79a..4e911403de 100644 --- a/backend/.sqlx/query-ff0403790674cdb07022af71c2377afbd8b3a660b3be27514b517c077c63c238.json +++ b/backend/.sqlx/query-ff0403790674cdb07022af71c2377afbd8b3a660b3be27514b517c077c63c238.json @@ -65,7 +65,8 @@ "csharp", "oracledb", "nu", - "java" + "java", + "duckdb" ] } } diff --git a/backend/.vscode/settings.json b/backend/.vscode/settings.json index 00cebd7fb6..6c9612d5c3 100644 --- a/backend/.vscode/settings.json +++ b/backend/.vscode/settings.json @@ -12,5 +12,6 @@ "conventionalCommits.scopes": [ "restructring triggers, decoding trigger message on work" ], - "rust-analyzer.cargo.features": ["postgres_trigger"] + "files.exclude": { "**/*ee.rs": false }, + "search.exclude": { "**/*ee.rs": false } } diff --git a/backend/Cargo.lock b/backend/Cargo.lock index fe8e9c769b..24fb8436bb 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -29,9 +29,9 @@ checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" [[package]] name = "adler2" -version = "2.0.0" +version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "512761e0bb2578dd7380c6baaa0f4ce03e84f95e960231d1dec8bf4d7d6e2627" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" [[package]] name = "aead" @@ -175,9 +175,9 @@ dependencies = [ [[package]] name = "anstream" -version = "0.6.18" +version = "0.6.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8acc5369981196006228e28809f761875c0327210a891e941f4c683b3a99529b" +checksum = "301af1932e46185686725e0fad2f8f2aa7da69dd70bf6ecc44d6b703844a3933" dependencies = [ "anstyle", "anstyle-parse", @@ -190,33 +190,33 @@ dependencies = [ [[package]] name = "anstyle" -version = "1.0.10" +version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "55cc3b69f167a1ef2e161439aa98aed94e6028e5f9a59be9a6ffb47aef1651f9" +checksum = "862ed96ca487e809f1c8e5a8447f6ee2cf102f846893800b20cebdf541fc6bbd" [[package]] name = "anstyle-parse" -version = "0.2.6" +version = "0.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b2d16507662817a6a20a9ea92df6652ee4f94f914589377d69f3b21bc5798a9" +checksum = "4e7644824f0aa2c7b9384579234ef10eb7efb6a0deb83f9630a49594dd9c15c2" dependencies = [ "utf8parse", ] [[package]] name = "anstyle-query" -version = "1.1.2" +version = "1.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79947af37f4177cfead1110013d678905c37501914fba0efea834c3fe9a8d60c" +checksum = "6c8bdeb6047d8983be085bab0ba1472e6dc604e7041dbf6fcd5e71523014fae9" dependencies = [ "windows-sys 0.59.0", ] [[package]] name = "anstyle-wincon" -version = "3.0.8" +version = "3.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6680de5231bd6ee4c6191b8a1325daa282b415391ec9d3a37bd34f2060dc73fa" +checksum = "403f75924867bb1033c59fbf0797484329750cfbe3c4325cd33127941fabc882" dependencies = [ "anstyle", "once_cell_polyfill", @@ -286,57 +286,25 @@ dependencies = [ "serde", ] -[[package]] -name = "arrow" -version = "54.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5ec52ba94edeed950e4a41f75d35376df196e8cb04437f7280a5aa49f20f796" -dependencies = [ - "arrow-arith 54.3.1", - "arrow-array 54.3.1", - "arrow-buffer 54.3.1", - "arrow-cast 54.3.1", - "arrow-data 54.3.1", - "arrow-ord 54.3.1", - "arrow-row 54.3.1", - "arrow-schema 54.3.1", - "arrow-select 54.3.1", - "arrow-string 54.3.1", -] - [[package]] name = "arrow" version = "55.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b1bb018b6960c87fd9d025009820406f74e83281185a8bdcb44880d2aa5c9a87" dependencies = [ - "arrow-arith 55.1.0", - "arrow-array 55.1.0", - "arrow-buffer 55.1.0", - "arrow-cast 55.1.0", + "arrow-arith", + "arrow-array", + "arrow-buffer", + "arrow-cast", "arrow-csv", - "arrow-data 55.1.0", + "arrow-data", "arrow-ipc", "arrow-json", - "arrow-ord 55.1.0", - "arrow-row 55.1.0", - "arrow-schema 55.1.0", - "arrow-select 55.1.0", - "arrow-string 55.1.0", -] - -[[package]] -name = "arrow-arith" -version = "54.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fc766fdacaf804cb10c7c70580254fcdb5d55cdfda2bc57b02baf5223a3af9e" -dependencies = [ - "arrow-array 54.3.1", - "arrow-buffer 54.3.1", - "arrow-data 54.3.1", - "arrow-schema 54.3.1", - "chrono", - "num", + "arrow-ord", + "arrow-row", + "arrow-schema", + "arrow-select", + "arrow-string", ] [[package]] @@ -345,30 +313,14 @@ version = "55.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "44de76b51473aa888ecd6ad93ceb262fb8d40d1f1154a4df2f069b3590aa7575" dependencies = [ - "arrow-array 55.1.0", - "arrow-buffer 55.1.0", - "arrow-data 55.1.0", - "arrow-schema 55.1.0", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", "chrono", "num", ] -[[package]] -name = "arrow-array" -version = "54.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a12fcdb3f1d03f69d3ec26ac67645a8fe3f878d77b5ebb0b15d64a116c212985" -dependencies = [ - "ahash 0.8.12", - "arrow-buffer 54.3.1", - "arrow-data 54.3.1", - "arrow-schema 54.3.1", - "chrono", - "half", - "hashbrown 0.15.3", - "num", -] - [[package]] name = "arrow-array" version = "55.1.0" @@ -376,24 +328,13 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "29ed77e22744475a9a53d00026cf8e166fe73cf42d89c4c4ae63607ee1cfcc3f" dependencies = [ "ahash 0.8.12", - "arrow-buffer 55.1.0", - "arrow-data 55.1.0", - "arrow-schema 55.1.0", + "arrow-buffer", + "arrow-data", + "arrow-schema", "chrono", "chrono-tz", "half", - "hashbrown 0.15.3", - "num", -] - -[[package]] -name = "arrow-buffer" -version = "54.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "263f4801ff1839ef53ebd06f99a56cecd1dbaf314ec893d93168e2e860e0291c" -dependencies = [ - "bytes", - "half", + "hashbrown 0.15.4", "num", ] @@ -408,38 +349,17 @@ dependencies = [ "num", ] -[[package]] -name = "arrow-cast" -version = "54.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ede6175fbc039dfc946a61c1b6d42fd682fcecf5ab5d148fbe7667705798cac9" -dependencies = [ - "arrow-array 54.3.1", - "arrow-buffer 54.3.1", - "arrow-data 54.3.1", - "arrow-schema 54.3.1", - "arrow-select 54.3.1", - "atoi", - "base64 0.22.1", - "chrono", - "comfy-table", - "half", - "lexical-core", - "num", - "ryu", -] - [[package]] name = "arrow-cast" version = "55.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f39e1d774ece9292697fcbe06b5584401b26bd34be1bec25c33edae65c2420ff" dependencies = [ - "arrow-array 55.1.0", - "arrow-buffer 55.1.0", - "arrow-data 55.1.0", - "arrow-schema 55.1.0", - "arrow-select 55.1.0", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", "atoi", "base64 0.22.1", "chrono", @@ -456,9 +376,9 @@ version = "55.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9055c972a07bf12c2a827debfd34f88d3b93da1941d36e1d9fee85eebe38a12a" dependencies = [ - "arrow-array 55.1.0", - "arrow-cast 55.1.0", - "arrow-schema 55.1.0", + "arrow-array", + "arrow-cast", + "arrow-schema", "chrono", "csv", "csv-core", @@ -466,26 +386,14 @@ dependencies = [ "regex", ] -[[package]] -name = "arrow-data" -version = "54.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61cfdd7d99b4ff618f167e548b2411e5dd2c98c0ddebedd7df433d34c20a4429" -dependencies = [ - "arrow-buffer 54.3.1", - "arrow-schema 54.3.1", - "half", - "num", -] - [[package]] name = "arrow-data" version = "55.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf75ac27a08c7f48b88e5c923f267e980f27070147ab74615ad85b5c5f90473d" dependencies = [ - "arrow-buffer 55.1.0", - "arrow-schema 55.1.0", + "arrow-buffer", + "arrow-schema", "half", "num", ] @@ -496,10 +404,10 @@ version = "55.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a222f0d93772bd058d1268f4c28ea421a603d66f7979479048c429292fac7b2e" dependencies = [ - "arrow-array 55.1.0", - "arrow-buffer 55.1.0", - "arrow-data 55.1.0", - "arrow-schema 55.1.0", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", "flatbuffers", "lz4_flex", ] @@ -510,11 +418,11 @@ version = "55.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9085342bbca0f75e8cb70513c0807cc7351f1fbf5cb98192a67d5e3044acb033" dependencies = [ - "arrow-array 55.1.0", - "arrow-buffer 55.1.0", - "arrow-cast 55.1.0", - "arrow-data 55.1.0", - "arrow-schema 55.1.0", + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-data", + "arrow-schema", "chrono", "half", "indexmap 2.9.0", @@ -526,43 +434,17 @@ dependencies = [ "simdutf8", ] -[[package]] -name = "arrow-ord" -version = "54.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0a3334a743bd2a1479dbc635540617a3923b4b2f6870f37357339e6b5363c21" -dependencies = [ - "arrow-array 54.3.1", - "arrow-buffer 54.3.1", - "arrow-data 54.3.1", - "arrow-schema 54.3.1", - "arrow-select 54.3.1", -] - [[package]] name = "arrow-ord" version = "55.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ab2f1065a5cad7b9efa9e22ce5747ce826aa3855766755d4904535123ef431e7" dependencies = [ - "arrow-array 55.1.0", - "arrow-buffer 55.1.0", - "arrow-data 55.1.0", - "arrow-schema 55.1.0", - "arrow-select 55.1.0", -] - -[[package]] -name = "arrow-row" -version = "54.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d1d7a7291d2c5107e92140f75257a99343956871f3d3ab33a7b41532f79cb68" -dependencies = [ - "arrow-array 54.3.1", - "arrow-buffer 54.3.1", - "arrow-data 54.3.1", - "arrow-schema 54.3.1", - "half", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", ] [[package]] @@ -571,40 +453,20 @@ version = "55.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3703a0e3e92d23c3f756df73d2dc9476873f873a76ae63ef9d3de17fda83b2d8" dependencies = [ - "arrow-array 55.1.0", - "arrow-buffer 55.1.0", - "arrow-data 55.1.0", - "arrow-schema 55.1.0", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", "half", ] -[[package]] -name = "arrow-schema" -version = "54.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39cfaf5e440be44db5413b75b72c2a87c1f8f0627117d110264048f2969b99e9" -dependencies = [ - "bitflags 2.9.1", -] - [[package]] name = "arrow-schema" version = "55.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "73a47aa0c771b5381de2b7f16998d351a6f4eb839f1e13d48353e17e873d969b" - -[[package]] -name = "arrow-select" -version = "54.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69efcd706420e52cd44f5c4358d279801993846d1c2a8e52111853d61d55a619" dependencies = [ - "ahash 0.8.12", - "arrow-array 54.3.1", - "arrow-buffer 54.3.1", - "arrow-data 54.3.1", - "arrow-schema 54.3.1", - "num", + "bitflags 2.9.1", ] [[package]] @@ -614,41 +476,24 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24b7b85575702b23b85272b01bc1c25a01c9b9852305e5d0078c79ba25d995d4" dependencies = [ "ahash 0.8.12", - "arrow-array 55.1.0", - "arrow-buffer 55.1.0", - "arrow-data 55.1.0", - "arrow-schema 55.1.0", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", "num", ] -[[package]] -name = "arrow-string" -version = "54.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a21546b337ab304a32cfc0770f671db7411787586b45b78b4593ae78e64e2b03" -dependencies = [ - "arrow-array 54.3.1", - "arrow-buffer 54.3.1", - "arrow-data 54.3.1", - "arrow-schema 54.3.1", - "arrow-select 54.3.1", - "memchr", - "num", - "regex", - "regex-syntax 0.8.5", -] - [[package]] name = "arrow-string" version = "55.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9260fddf1cdf2799ace2b4c2fc0356a9789fa7551e0953e35435536fecefebbd" dependencies = [ - "arrow-array 55.1.0", - "arrow-buffer 55.1.0", - "arrow-data 55.1.0", - "arrow-schema 55.1.0", - "arrow-select 55.1.0", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", "memchr", "num", "regex", @@ -712,7 +557,7 @@ dependencies = [ "proc-macro2", "quote", "swc_macros_common", - "syn 2.0.101", + "syn 2.0.103", ] [[package]] @@ -791,7 +636,7 @@ dependencies = [ "bytes", "http 1.3.1", "rand 0.8.5", - "reqwest 0.12.18", + "reqwest 0.12.20", "serde", "serde-aux", "serde_json", @@ -814,7 +659,7 @@ checksum = "3b43422f69d8ff38f95f1b2bb76517c91589a924d1559a0e935d7c8ce0274c11" dependencies = [ "proc-macro2", "quote", - "syn 2.0.101", + "syn 2.0.103", ] [[package]] @@ -836,7 +681,7 @@ checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.101", + "syn 2.0.103", ] [[package]] @@ -847,7 +692,7 @@ checksum = "e539d3fca749fcee5236ab05e93a52867dd549cc157c8cb7f99595f3cedffdb5" dependencies = [ "proc-macro2", "quote", - "syn 2.0.101", + "syn 2.0.103", ] [[package]] @@ -902,9 +747,9 @@ checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" [[package]] name = "aws-config" -version = "1.6.3" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02a18fd934af6ae7ca52410d4548b98eb895aab0f1ea417d168d85db1434a141" +checksum = "455e9fb7743c6f6267eb2830ccc08686fbb3d13c9a689369562fd4d4ef9ea462" dependencies = [ "aws-credential-types", "aws-runtime", @@ -967,9 +812,9 @@ dependencies = [ [[package]] name = "aws-runtime" -version = "1.5.7" +version = "1.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c4063282c69991e57faab9e5cb21ae557e59f5b0fb285c196335243df8dc25c" +checksum = "4f6c68419d8ba16d9a7463671593c54f81ba58cab466e9b759418da606dcc2e2" dependencies = [ "aws-credential-types", "aws-sigv4", @@ -991,9 +836,9 @@ dependencies = [ [[package]] name = "aws-sdk-sqs" -version = "1.71.0" +version = "1.73.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0519921fc7065a54b8dec1d10de2893d99a61d030b9f8eb00ee83eb62d2a2676" +checksum = "8f3d3cf0d52a50e0ac3fada7e0db27a2172b986f833d061def1df78c7dbe80ae" dependencies = [ "aws-credential-types", "aws-runtime", @@ -1013,9 +858,9 @@ dependencies = [ [[package]] name = "aws-sdk-sso" -version = "1.71.0" +version = "1.73.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95a4fd09d6e863655d99cd2260f271c6d1030dc6bfad68e19e126d2e4c8ceb18" +checksum = "b2ac1674cba7872061a29baaf02209fefe499ff034dfd91bd4cc59e4d7741489" dependencies = [ "aws-credential-types", "aws-runtime", @@ -1035,9 +880,9 @@ dependencies = [ [[package]] name = "aws-sdk-ssooidc" -version = "1.72.0" +version = "1.74.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3224ab02ebb3074467a33d57caf6fcb487ca36f3697fdd381b0428dc72380696" +checksum = "3a6a22f077f5fd3e3c0270d4e1a110346cddf6769e9433eb9e6daceb4ca3b149" dependencies = [ "aws-credential-types", "aws-runtime", @@ -1057,9 +902,9 @@ dependencies = [ [[package]] name = "aws-sdk-sts" -version = "1.72.0" +version = "1.74.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6933f189ed1255e78175fbd73fb200c0aae7240d220ed3346f567b0ddca3083" +checksum = "19d440e1d368759bd10df0dbdddbfff6473d7cd73e9d9ef2363dc9995ac2d711" dependencies = [ "aws-credential-types", "aws-runtime", @@ -1080,9 +925,9 @@ dependencies = [ [[package]] name = "aws-sigv4" -version = "1.3.2" +version = "1.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3734aecf9ff79aa401a6ca099d076535ab465ff76b46440cf567c8e70b65dc13" +checksum = "ddfb9021f581b71870a17eac25b52335b82211cdc092e02b6876b2bcefa61666" dependencies = [ "aws-credential-types", "aws-smithy-http", @@ -1133,13 +978,14 @@ dependencies = [ [[package]] name = "aws-smithy-http-client" -version = "1.0.2" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e44697a9bded898dcd0b1cb997430d949b87f4f8940d91023ae9062bf218250" +checksum = "7f491388e741b7ca73b24130ff464c1478acc34d5b331b7dd0a2ee4643595a15" dependencies = [ "aws-smithy-async", "aws-smithy-runtime-api", "aws-smithy-types", + "h2 0.3.26", "h2 0.4.10", "http 0.2.12", "http 1.3.1", @@ -1147,11 +993,11 @@ dependencies = [ "hyper 0.14.32", "hyper 1.6.0", "hyper-rustls 0.24.2", - "hyper-rustls 0.27.6", + "hyper-rustls 0.27.7", "hyper-util", "pin-project-lite", "rustls 0.21.12", - "rustls 0.23.27", + "rustls 0.23.28", "rustls-native-certs 0.8.1", "rustls-pki-types", "tokio", @@ -1161,9 +1007,9 @@ dependencies = [ [[package]] name = "aws-smithy-json" -version = "0.61.3" +version = "0.61.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92144e45819cae7dc62af23eac5a038a58aa544432d2102609654376a900bd07" +checksum = "a16e040799d29c17412943bdbf488fd75db04112d0c0d4b9290bacf5ae0014b9" dependencies = [ "aws-smithy-types", ] @@ -1213,9 +1059,9 @@ dependencies = [ [[package]] name = "aws-smithy-runtime-api" -version = "1.8.0" +version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1e5d9e3a80a18afa109391fb5ad09c3daf887b516c6fd805a157c6ea7994a57" +checksum = "bd8531b6d8882fd8f48f82a9754e682e29dd44cff27154af51fa3eb730f59efb" dependencies = [ "aws-smithy-async", "aws-smithy-types", @@ -1230,9 +1076,9 @@ dependencies = [ [[package]] name = "aws-smithy-types" -version = "1.3.1" +version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40076bd09fadbc12d5e026ae080d0930defa606856186e31d83ccc6a255eeaf3" +checksum = "d498595448e43de7f4296b7b7a18a8a02c61ec9349128c80a368f7c3b4ab11a8" dependencies = [ "base64-simd 0.8.0", "bytes", @@ -1266,9 +1112,9 @@ dependencies = [ [[package]] name = "aws-smithy-xml" -version = "0.60.9" +version = "0.60.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab0b0166827aa700d3dc519f72f8b3a91c35d0b8d042dc5d643a91e6f80648fc" +checksum = "3db87b96cb1b16c024980f133968d52882ca0daaee3a086c6decc500f6c99728" dependencies = [ "xmlparser", ] @@ -1369,7 +1215,7 @@ dependencies = [ "addr2line", "cfg-if", "libc", - "miniz_oxide 0.8.8", + "miniz_oxide 0.8.9", "object", "rustc-demangle", "windows-targets 0.52.6", @@ -1426,9 +1272,9 @@ dependencies = [ [[package]] name = "base64ct" -version = "1.7.3" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89e25b6adfb930f02d1981565a6e5d9c547ac15a96606256d3b59040e5cd4ca3" +checksum = "55248b47b0caf0546f7988906588779981c43bb1bc9d0c44087278f80cdb44ba" [[package]] name = "better_scoped_tls" @@ -1480,7 +1326,7 @@ dependencies = [ "regex", "rustc-hash 1.1.0", "shlex", - "syn 2.0.101", + "syn 2.0.103", "which 4.4.2", ] @@ -1501,7 +1347,7 @@ dependencies = [ "regex", "rustc-hash 1.1.0", "shlex", - "syn 2.0.101", + "syn 2.0.103", ] [[package]] @@ -1706,7 +1552,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.101", + "syn 2.0.103", ] [[package]] @@ -1716,7 +1562,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "17d4f95e880cfd28c4ca5a006cf7f6af52b4bcb7b5866f573b2faa126fb7affb" dependencies = [ "quote", - "syn 2.0.101", + "syn 2.0.103", ] [[package]] @@ -1793,9 +1639,9 @@ dependencies = [ [[package]] name = "bumpalo" -version = "3.17.0" +version = "3.18.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1628fb46dfa0b37568d12e5edd512553eccf6a22a78e8bde00bb4aed84d5bdbf" +checksum = "793db76d6187cd04dff33004d8e6c9cc4e05cd330500379d2394209271b4aeee" dependencies = [ "allocator-api2", ] @@ -1835,9 +1681,9 @@ dependencies = [ [[package]] name = "bytemuck" -version = "1.23.0" +version = "1.23.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9134a6ef01ce4b366b50689c94f82c14bc72bc5d0386829828a2e2752ef7958c" +checksum = "5c76a5792e44e4abe34d3abf15636779261d45a7450612059293d1d2cfc63422" dependencies = [ "bytemuck_derive", ] @@ -1850,7 +1696,7 @@ checksum = "7ecc273b49b3205b83d648f0690daa588925572cc5063745bfe547fe7ec8e1a1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.101", + "syn 2.0.103", ] [[package]] @@ -1993,7 +1839,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3b4a6cae9efc04cc6cbb8faf338d2c497c165c83e74509cf4dbedea948bbf6e5" dependencies = [ "quote", - "syn 2.0.101", + "syn 2.0.103", ] [[package]] @@ -2013,9 +1859,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.25" +version = "1.2.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0fc897dc1e865cc67c0e05a836d9d3f1df3cbe442aa4a9473b18e12624a4951" +checksum = "d487aa071b5f64da6f19a3e848e3578944b726ee5a4854b82172f02aa876bfdc" dependencies = [ "jobserver", "libc", @@ -2039,9 +1885,9 @@ dependencies = [ [[package]] name = "cfg-if" -version = "1.0.0" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +checksum = "9555578bc9e57714c812a1f84e4fc5b4d21fcb063490c624de019f7464c91268" [[package]] name = "cfg_aliases" @@ -2133,9 +1979,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.39" +version = "4.5.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd60e63e9be68e5fb56422e397cf9baddded06dae1d2e523401542383bc72a9f" +checksum = "40b6887a1d8685cebccf115538db5c0efe625ccac9696ad45c409d96566e910f" dependencies = [ "clap_builder", "clap_derive", @@ -2143,9 +1989,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.39" +version = "4.5.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89cc6392a1f72bbeb820d71f32108f61fdaf18bc526e1d23954168a67759ef51" +checksum = "e0c66c08ce9f0c698cbce5c0279d0bb6ac936d8674174fe48f736533b964f59e" dependencies = [ "anstream", "anstyle", @@ -2155,21 +2001,21 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.5.32" +version = "4.5.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09176aae279615badda0765c0c0b3f6ed53f4709118af73cf4655d85d1530cd7" +checksum = "d2c7947ae4cc3d851207c1adb5b5e260ff0cca11446b1d6d1423788e442257ce" dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.101", + "syn 2.0.103", ] [[package]] name = "clap_lex" -version = "0.7.4" +version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f46ad14479a25103f283c0f10005961cf086d8dc42205bb44c46ac563475dca6" +checksum = "b94f61472cee1439c0b966b47e3aca9ae07e45d070759512cd390ea2bebc6675" [[package]] name = "clipboard-win" @@ -2217,7 +2063,7 @@ dependencies = [ "nom 7.1.3", "proc-macro2", "quote", - "syn 2.0.101", + "syn 2.0.103", ] [[package]] @@ -2228,9 +2074,9 @@ checksum = "3d7b894f5411737b7867f4827955924d7c254fc9f4d91a6aad6b097804b1018b" [[package]] name = "colorchoice" -version = "1.0.3" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b63caa9aa9397e2d9480a9b13673856c78d8ac123288526c37d7839f2a86990" +checksum = "b05b61dc5112cbb17e4b6cd61790d9845d13888356391624cbe7e41efeac1e75" [[package]] name = "comfy-table" @@ -2239,7 +2085,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4a65ebfec4fb190b6f90e944a817d60499ee0744e582530e2c9900a22e591d9a" dependencies = [ "unicode-segmentation", - "unicode-width 0.2.0", + "unicode-width 0.2.1", ] [[package]] @@ -2266,7 +2112,7 @@ dependencies = [ "encode_unicode", "libc", "once_cell", - "unicode-width 0.2.0", + "unicode-width 0.2.1", "windows-sys 0.59.0", ] @@ -2599,7 +2445,7 @@ checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.101", + "syn 2.0.103", ] [[package]] @@ -2682,7 +2528,7 @@ dependencies = [ "proc-macro2", "quote", "strsim 0.11.1", - "syn 2.0.101", + "syn 2.0.103", ] [[package]] @@ -2715,7 +2561,7 @@ checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" dependencies = [ "darling_core 0.20.11", "quote", - "syn 2.0.101", + "syn 2.0.103", ] [[package]] @@ -2763,9 +2609,9 @@ version = "47.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ffe060b978f74ab446be722adb8a274e052e005bf6dfd171caadc3abaad10080" dependencies = [ - "arrow 55.1.0", + "arrow", "arrow-ipc", - "arrow-schema 55.1.0", + "arrow-schema", "async-trait", "bytes", "bzip2", @@ -2818,7 +2664,7 @@ version = "47.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "61fe34f401bd03724a1f96d12108144f8cd495a3cdda2bf5e091822fb80b7e66" dependencies = [ - "arrow 55.1.0", + "arrow", "async-trait", "dashmap 6.1.0", "datafusion-common", @@ -2844,7 +2690,7 @@ version = "47.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a4411b8e3bce5e0fc7521e44f201def2e2d5d1b5f176fb56e8cdc9942c890f00" dependencies = [ - "arrow 55.1.0", + "arrow", "async-trait", "datafusion-catalog", "datafusion-common", @@ -2868,7 +2714,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0734015d81c8375eb5d4869b7f7ecccc2ee8d6cb81948ef737cd0e7b743bd69c" dependencies = [ "ahash 0.8.12", - "arrow 55.1.0", + "arrow", "arrow-ipc", "base64 0.22.1", "half", @@ -2902,7 +2748,7 @@ version = "47.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "04e602dcdf2f50c2abf297cc2203c73531e6f48b29516af7695d338cf2a778b1" dependencies = [ - "arrow 55.1.0", + "arrow", "async-compression", "async-trait", "bytes", @@ -2938,7 +2784,7 @@ version = "47.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3bb2253952dc32296ed5b84077cb2e0257fea4be6373e1c376426e17ead4ef6" dependencies = [ - "arrow 55.1.0", + "arrow", "async-trait", "bytes", "datafusion-catalog", @@ -2963,7 +2809,7 @@ version = "47.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5b8c7f47a5d2fe03bfa521ec9bafdb8a5c82de8377f60967c3663f00c8790352" dependencies = [ - "arrow 55.1.0", + "arrow", "async-trait", "bytes", "datafusion-catalog", @@ -2988,7 +2834,7 @@ version = "47.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "27d15868ea39ed2dc266728b554f6304acd473de2142281ecfa1294bb7415923" dependencies = [ - "arrow 55.1.0", + "arrow", "async-trait", "bytes", "datafusion-catalog", @@ -3025,7 +2871,7 @@ version = "47.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06f004d100f49a3658c9da6fb0c3a9b760062d96cd4ad82ccc3b7b69a9fb2f84" dependencies = [ - "arrow 55.1.0", + "arrow", "dashmap 6.1.0", "datafusion-common", "datafusion-expr", @@ -3044,7 +2890,7 @@ version = "47.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7a4e4ce3802609be38eeb607ee72f6fe86c3091460de9dbfae9e18db423b3964" dependencies = [ - "arrow 55.1.0", + "arrow", "chrono", "datafusion-common", "datafusion-doc", @@ -3065,7 +2911,7 @@ version = "47.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "422ac9cf3b22bbbae8cdf8ceb33039107fde1b5492693168f13bd566b1bcc839" dependencies = [ - "arrow 55.1.0", + "arrow", "datafusion-common", "indexmap 2.9.0", "itertools 0.14.0", @@ -3078,8 +2924,8 @@ version = "47.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2ddf0a0a2db5d2918349c978d42d80926c6aa2459cd8a3c533a84ec4bb63479e" dependencies = [ - "arrow 55.1.0", - "arrow-buffer 55.1.0", + "arrow", + "arrow-buffer", "base64 0.22.1", "blake2", "blake3", @@ -3108,7 +2954,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "408a05dafdc70d05a38a29005b8b15e21b0238734dab1e98483fcb58038c5aba" dependencies = [ "ahash 0.8.12", - "arrow 55.1.0", + "arrow", "datafusion-common", "datafusion-doc", "datafusion-execution", @@ -3129,7 +2975,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "756d21da2dd6c9bef97af1504970ff56cbf35d03fbd4ffd62827f02f4d2279d4" dependencies = [ "ahash 0.8.12", - "arrow 55.1.0", + "arrow", "datafusion-common", "datafusion-expr-common", "datafusion-physical-expr-common", @@ -3141,8 +2987,8 @@ version = "47.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8d8d50f6334b378930d992d801a10ac5b3e93b846b39e4a05085742572844537" dependencies = [ - "arrow 55.1.0", - "arrow-ord 55.1.0", + "arrow", + "arrow-ord", "datafusion-common", "datafusion-doc", "datafusion-execution", @@ -3162,7 +3008,7 @@ version = "47.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cc9a97220736c8fff1446e936be90d57216c06f28969f9ffd3b72ac93c958c8a" dependencies = [ - "arrow 55.1.0", + "arrow", "async-trait", "datafusion-catalog", "datafusion-common", @@ -3207,7 +3053,7 @@ checksum = "df6f88d7ee27daf8b108ba910f9015176b36fbc72902b1ca5c2a5f1d1717e1a1" dependencies = [ "datafusion-expr", "quote", - "syn 2.0.101", + "syn 2.0.103", ] [[package]] @@ -3216,7 +3062,7 @@ version = "47.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "084d9f979c4b155346d3c34b18f4256e6904ded508e9554d90fed416415c3515" dependencies = [ - "arrow 55.1.0", + "arrow", "chrono", "datafusion-common", "datafusion-expr", @@ -3236,7 +3082,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "64c536062b0076f4e30084065d805f389f9fe38af0ca75bcbac86bc5e9fbab65" dependencies = [ "ahash 0.8.12", - "arrow 55.1.0", + "arrow", "datafusion-common", "datafusion-expr", "datafusion-expr-common", @@ -3258,7 +3104,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8a92b53b3193fac1916a1c5b8e3f4347c526f6822e56b71faa5fb372327a863" dependencies = [ "ahash 0.8.12", - "arrow 55.1.0", + "arrow", "datafusion-common", "datafusion-expr-common", "hashbrown 0.14.5", @@ -3271,7 +3117,7 @@ version = "47.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6fa0a5ac94c7cf3da97bedabd69d6bbca12aef84b9b37e6e9e8c25286511b5e2" dependencies = [ - "arrow 55.1.0", + "arrow", "datafusion-common", "datafusion-execution", "datafusion-expr", @@ -3291,9 +3137,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "690c615db468c2e5fe5085b232d8b1c088299a6c63d87fd960a354a71f7acb55" dependencies = [ "ahash 0.8.12", - "arrow 55.1.0", - "arrow-ord 55.1.0", - "arrow-schema 55.1.0", + "arrow", + "arrow-ord", + "arrow-schema", "async-trait", "chrono", "datafusion-common", @@ -3320,7 +3166,7 @@ version = "47.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ad229a134c7406c057ece00c8743c0c34b97f4e72f78b475fe17b66c5e14fa4f" dependencies = [ - "arrow 55.1.0", + "arrow", "async-trait", "dashmap 6.1.0", "datafusion-common", @@ -3344,7 +3190,7 @@ version = "47.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "64f6ab28b72b664c21a27b22a2ff815fd390ed224c26e89a93b5a8154a4e8607" dependencies = [ - "arrow 55.1.0", + "arrow", "bigdecimal", "datafusion-common", "datafusion-expr", @@ -3385,7 +3231,7 @@ dependencies = [ "once_cell", "percent-encoding", "serde", - "sourcemap 9.2.1", + "sourcemap 9.2.2", "swc_atoms", "swc_common", "swc_config", @@ -3645,7 +3491,7 @@ checksum = "babccedee31ce7e57c3e6dff2cb3ab8d68c49d0df8222fe0d11d628e65192790" dependencies = [ "proc-macro2", "quote", - "syn 2.0.101", + "syn 2.0.103", ] [[package]] @@ -3670,7 +3516,7 @@ dependencies = [ "http 1.3.1", "http-body-util", "hyper 1.6.0", - "hyper-rustls 0.27.6", + "hyper-rustls 0.27.7", "hyper-util", "ipnet", "percent-encoding", @@ -4036,7 +3882,7 @@ dependencies = [ "stringcase", "strum 0.25.0", "strum_macros 0.25.3", - "syn 2.0.101", + "syn 2.0.103", "thiserror 2.0.12", ] @@ -4276,7 +4122,7 @@ dependencies = [ "deno_tls", "http-body-util", "hyper 1.6.0", - "hyper-rustls 0.27.6", + "hyper-rustls 0.27.7", "hyper-util", "log", "once_cell", @@ -4310,7 +4156,7 @@ dependencies = [ "deno_core", "deno_error", "deno_native_certs", - "rustls 0.23.27", + "rustls 0.23.28", "rustls-pemfile 2.2.0", "rustls-tokio-stream", "rustls-webpki 0.102.8", @@ -4322,11 +4168,11 @@ dependencies = [ [[package]] name = "deno_unsync" -version = "0.4.3" +version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "47c618b51088b3ac67f15c69b3ed7620ba3a7d495e5a090186df9424b5ab623e" +checksum = "6742a724e8becb372a74c650a1aefb8924a5b8107f7d75b3848763ea24b27a87" dependencies = [ - "futures", + "futures-util", "parking_lot 0.12.4", "tokio", ] @@ -4538,7 +4384,7 @@ checksum = "8034092389675178f570469e6c3b0465d3d30b4505c294a6550db47f3c17ad18" dependencies = [ "proc-macro2", "quote", - "syn 2.0.101", + "syn 2.0.103", ] [[package]] @@ -4559,7 +4405,7 @@ checksum = "30542c1ad912e0e3d22a1935c290e12e8a29d704a420177a31faad4a601a0800" dependencies = [ "proc-macro2", "quote", - "syn 2.0.101", + "syn 2.0.103", ] [[package]] @@ -4603,7 +4449,7 @@ dependencies = [ "proc-macro2", "quote", "rustc_version 0.4.1", - "syn 2.0.101", + "syn 2.0.103", ] [[package]] @@ -4708,7 +4554,7 @@ checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.101", + "syn 2.0.103", ] [[package]] @@ -4743,7 +4589,7 @@ checksum = "788160fb30de9cdd857af31c6a2675904b16ece8fc2737b2c7127ba368c9d0f4" dependencies = [ "proc-macro2", "quote", - "syn 2.0.101", + "syn 2.0.103", ] [[package]] @@ -4806,17 +4652,16 @@ dependencies = [ [[package]] name = "duckdb" -version = "1.2.2" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49ac283b6621e3becf8014d1efa655522794075834c72f744573debef9c9f6c8" +checksum = "379464d1fbb69d1bbda7c11d96fafde6f2bdb78c67e630e6339b048e6e45107e" dependencies = [ - "arrow 54.3.1", + "arrow", "cast", "fallible-iterator 0.3.0", "fallible-streaming-iterator", "hashlink 0.9.1", "libduckdb-sys", - "memchr", "num-integer", "rust_decimal", "smallvec", @@ -5019,27 +4864,27 @@ dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.101", + "syn 2.0.103", ] [[package]] name = "enumflags2" -version = "0.7.11" +version = "0.7.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba2f4b465f5318854c6f8dd686ede6c0a9dc67d4b1ac241cf0eb51521a309147" +checksum = "1027f7680c853e056ebcec683615fb6fbbc07dbaa13b4d5d9442b146ded4ecef" dependencies = [ "enumflags2_derive", ] [[package]] name = "enumflags2_derive" -version = "0.7.11" +version = "0.7.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc4caf64a58d7a6d65ab00639b046ff54399a39f5f2554728895ace4b297cd79" +checksum = "67c78a4d8fdf9953a5c9d458f9efe940fd97a0cab0941c075a813ac594733827" dependencies = [ "proc-macro2", "quote", - "syn 2.0.101", + "syn 2.0.103", ] [[package]] @@ -5270,14 +5115,14 @@ dependencies = [ [[package]] name = "flate2" -version = "1.1.1" +version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ced92e76e966ca2fd84c8f7aa01a4aea65b0eb6648d72f7c8f3e2764a67fece" +checksum = "4a3d7db9596fecd151c5f638c0ee5d5bd487b6e0ea232e5dc96d5250f6f94b1d" dependencies = [ "crc32fast", "libz-rs-sys", "libz-sys", - "miniz_oxide 0.8.8", + "miniz_oxide 0.8.9", ] [[package]] @@ -5340,7 +5185,7 @@ checksum = "1a5c6c585bc94aaf2c7b51dd4c2ba22680844aba4c687be581871a6f518c5742" dependencies = [ "proc-macro2", "quote", - "syn 2.0.101", + "syn 2.0.103", ] [[package]] @@ -5378,7 +5223,7 @@ checksum = "32016f1242eb82af5474752d00fd8ebcd9004bd69b462b1c91de833972d08ed4" dependencies = [ "proc-macro2", "swc_macros_common", - "syn 2.0.101", + "syn 2.0.103", ] [[package]] @@ -5513,7 +5358,7 @@ checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" dependencies = [ "proc-macro2", "quote", - "syn 2.0.101", + "syn 2.0.103", ] [[package]] @@ -5819,7 +5664,7 @@ dependencies = [ "libc", "log", "rustversion", - "windows 0.61.1", + "windows 0.61.3", ] [[package]] @@ -5845,11 +5690,11 @@ dependencies = [ [[package]] name = "getopts" -version = "0.2.21" +version = "0.2.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14dbbfd5c71d70241ecf9e6f13737f7b5ce823821063188d7e46c41d371eebd5" +checksum = "cba6ae63eb948698e300f645f87c70f76630d505f23b8907cf1e193ee85048c1" dependencies = [ - "unicode-width 0.1.14", + "unicode-width 0.2.1", ] [[package]] @@ -5861,7 +5706,7 @@ dependencies = [ "cfg-if", "js-sys", "libc", - "wasi 0.11.0+wasi-snapshot-preview1", + "wasi 0.11.1+wasi-snapshot-preview1", "wasm-bindgen", ] @@ -5912,7 +5757,7 @@ checksum = "53010ccb100b96a67bc32c0175f0ed1426b31b655d562898e57325f81c023ac0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.101", + "syn 2.0.103", ] [[package]] @@ -5990,7 +5835,7 @@ dependencies = [ "google-cloud-token", "home", "jsonwebtoken 9.3.1", - "reqwest 0.12.18", + "reqwest 0.12.20", "serde", "serde_json", "thiserror 1.0.69", @@ -6033,7 +5878,7 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d901aeb453fd80e51d64df4ee005014f6cf39f2d736dd64f7239c132d9d39a6a" dependencies = [ - "reqwest 0.12.18", + "reqwest 0.12.20", "thiserror 1.0.69", "tokio", ] @@ -6105,7 +5950,7 @@ checksum = "b89c83349105e3732062a895becfc71a8f921bb71ecbbdd8ff99263e3b53a0ca" dependencies = [ "bitflags 2.9.1", "gpu-descriptor-types", - "hashbrown 0.15.3", + "hashbrown 0.15.4", ] [[package]] @@ -6220,9 +6065,9 @@ dependencies = [ [[package]] name = "hashbrown" -version = "0.15.3" +version = "0.15.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84b26c544d002229e640969970a2e74021aadf6e2f96372b9c58eff97de08eb3" +checksum = "5971ac85611da7067dbfcabef3c70ebb5606018acd9e2a3903a0da507521e0d5" dependencies = [ "allocator-api2", "equivalent", @@ -6237,7 +6082,7 @@ checksum = "f208758247e68e239acaa059e72e4ce1f30f2a4b6523f19c1b923d25b7e9cceb" dependencies = [ "proc-macro2", "quote", - "syn 2.0.101", + "syn 2.0.103", ] [[package]] @@ -6255,7 +6100,7 @@ version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7382cf6263419f2d8df38c55d7da83da5c18aef87fc7a7fc1fb1e344edfe14c1" dependencies = [ - "hashbrown 0.15.3", + "hashbrown 0.15.4", ] [[package]] @@ -6286,9 +6131,9 @@ checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" [[package]] name = "hermit-abi" -version = "0.5.1" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f154ce46856750ed433c8649605bf7ed2de3bc35fd9d2a9f30cddd873c80cb08" +checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c" [[package]] name = "hex" @@ -6577,14 +6422,14 @@ dependencies = [ [[package]] name = "hyper-rustls" -version = "0.27.6" +version = "0.27.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03a01595e11bdcec50946522c32dde3fc6914743000a68b93000965f2f02406d" +checksum = "e3c93eb611681b207e1fe55d5a71ecf91572ec8a6705cdb6857f7d8d5242cf58" dependencies = [ "http 1.3.1", "hyper 1.6.0", "hyper-util", - "rustls 0.23.27", + "rustls 0.23.28", "rustls-native-certs 0.8.1", "rustls-pki-types", "tokio", @@ -6637,9 +6482,9 @@ dependencies = [ [[package]] name = "hyper-util" -version = "0.1.13" +version = "0.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1c293b6b3d21eca78250dc7dbebd6b9210ec5530e038cbfe0661b5c47ab06e8" +checksum = "dc2fdfdbff08affe55bb779f33b053aa1fe5dd5b54c257343c17edfa55711bdb" dependencies = [ "base64 0.22.1", "bytes", @@ -6883,7 +6728,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cea70ddb795996207ad57735b50c5982d8844f38ba9ee5f1aedcfb708a2aa11e" dependencies = [ "equivalent", - "hashbrown 0.15.3", + "hashbrown 0.15.4", "serde", ] @@ -6896,7 +6741,7 @@ dependencies = [ "console", "number_prefix", "portable-atomic", - "unicode-width 0.2.0", + "unicode-width 0.2.1", "web-time", ] @@ -7003,7 +6848,7 @@ dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.101", + "syn 2.0.103", ] [[package]] @@ -7253,7 +7098,7 @@ dependencies = [ "proc-macro2", "quote", "regex", - "syn 2.0.101", + "syn 2.0.103", ] [[package]] @@ -7343,15 +7188,15 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.172" +version = "0.2.173" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d750af042f7ef4f724306de029d18836c26c1765a54a6a3f094cbd23a7267ffa" +checksum = "d8cfeafaffdbc32176b64fb251369d52ea9f0a8fbc6f8759edffef7b525d64bb" [[package]] name = "libduckdb-sys" -version = "1.2.2" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12cac9d03484c43fefac8b2066a253c9b0b3b0cd02cbe02a9ea2312f7e382618" +checksum = "25d3f1defe457d1ac0fbaef0fe6926953cb33419dd3c8dbac53882d020bee697" dependencies = [ "autocfg", "cc", @@ -7399,7 +7244,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "07033963ba89ebaf1584d767badaa2e8fcec21aedea6b8c0346d487d49c28667" dependencies = [ "cfg-if", - "windows-targets 0.53.0", + "windows-targets 0.53.2", ] [[package]] @@ -7427,7 +7272,7 @@ checksum = "c0ff37bd590ca25063e35af745c343cb7a0271906fb7b37e4813e8f79f00268d" dependencies = [ "bitflags 2.9.1", "libc", - "redox_syscall 0.5.12", + "redox_syscall 0.5.13", ] [[package]] @@ -7453,10 +7298,20 @@ dependencies = [ ] [[package]] -name = "libz-rs-sys" -version = "0.5.0" +name = "libyml" +version = "0.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6489ca9bd760fe9642d7644e827b0c9add07df89857b0416ee15c1cc1a3b8c5a" +checksum = "3302702afa434ffa30847a83305f0a69d6abd74293b6554c18ec85c7ef30c980" +dependencies = [ + "anyhow", + "version_check", +] + +[[package]] +name = "libz-rs-sys" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "172a788537a2221661b480fee8dc5f96c580eb34fa88764d3205dc356c7e4221" dependencies = [ "zlib-rs", ] @@ -7548,7 +7403,7 @@ version = "0.12.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "234cf4f4a04dc1f57e24b96cc0cd600cf2af460d4161ac5ecdd0af8e1f3b2a38" dependencies = [ - "hashbrown 0.15.3", + "hashbrown 0.15.4", ] [[package]] @@ -7557,7 +7412,7 @@ version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9f8cc7106155f10bdf99a6f379688f543ad6596a415375b36a59a054ceda1198" dependencies = [ - "hashbrown 0.15.3", + "hashbrown 0.15.4", ] [[package]] @@ -7577,11 +7432,11 @@ dependencies = [ [[package]] name = "lz4_flex" -version = "0.11.3" +version = "0.11.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75761162ae2b0e580d7e7c390558127e5f01b4194debd6221fd8c207fc80e3f5" +checksum = "2c592ad9fbc1b7838633b3ae55ce69b17d01150c72fcef229fbb819d39ee51ee" dependencies = [ - "twox-hash 1.6.3", + "twox-hash 2.1.1", ] [[package]] @@ -7606,9 +7461,9 @@ dependencies = [ [[package]] name = "macro_rules_attribute" -version = "0.2.0" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a82271f7bc033d84bbca59a3ce3e4159938cb08a9c3aebbe54d215131518a13" +checksum = "65049d7923698040cd0b1ddcced9b0eb14dd22c5f86ae59c3740eab64a676520" dependencies = [ "macro_rules_attribute-proc_macro", "paste", @@ -7616,9 +7471,9 @@ dependencies = [ [[package]] name = "macro_rules_attribute-proc_macro" -version = "0.2.0" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8dd856d451cc0da70e2ef2ce95a18e39a93b7558bedf10201ad28503f918568" +checksum = "670fdfda89751bc4a84ac13eaa63e205cf0fd22b4c9a5fbfa085b63c1f1d3a30" [[package]] name = "magic-crypt" @@ -7666,7 +7521,7 @@ dependencies = [ "base64 0.22.1", "gethostname", "mail-builder", - "rustls 0.23.27", + "rustls 0.23.28", "rustls-pki-types", "smtp-proto", "tokio", @@ -7810,9 +7665,9 @@ dependencies = [ [[package]] name = "memchr" -version = "2.7.4" +version = "2.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" +checksum = "32a282da65faaf38286cf3be983213fcf1d2e2a58700e808f83f4ea9a4804bc0" [[package]] name = "memmap2" @@ -7888,7 +7743,7 @@ checksum = "db5b29714e950dbb20d5e6f74f9dcec4edbcc1067bb7f8ed198c097b8c1a818b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.101", + "syn 2.0.103", ] [[package]] @@ -7934,9 +7789,9 @@ dependencies = [ [[package]] name = "miniz_oxide" -version = "0.8.8" +version = "0.8.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3be647b768db090acb35d5ec5db2b0e1f1de11133ca123b9eacf5137868f892a" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" dependencies = [ "adler2", "simd-adler32", @@ -7950,7 +7805,7 @@ checksum = "a4a650543ca06a924e8b371db273b2756685faae30f8487da1b56505a8f78b0c" dependencies = [ "libc", "log", - "wasi 0.11.0+wasi-snapshot-preview1", + "wasi 0.11.1+wasi-snapshot-preview1", "windows-sys 0.48.0", ] @@ -7961,7 +7816,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "78bed444cc8a2160f01cbcf811ef18cac863ad68ae8ca62092e8db51d51c761c" dependencies = [ "libc", - "wasi 0.11.0+wasi-snapshot-preview1", + "wasi 0.11.1+wasi-snapshot-preview1", "windows-sys 0.59.0", ] @@ -8008,7 +7863,7 @@ checksum = "c402a4092d5e204f32c9e155431046831fa712637043c58cb73bc6bc6c9663b5" dependencies = [ "proc-macro2", "quote", - "syn 2.0.101", + "syn 2.0.103", ] [[package]] @@ -8053,7 +7908,7 @@ dependencies = [ "proc-macro-error2", "proc-macro2", "quote", - "syn 2.0.101", + "syn 2.0.103", "termcolor", "thiserror 2.0.12", ] @@ -8084,7 +7939,7 @@ dependencies = [ "tokio", "tokio-native-tls", "tokio-util", - "twox-hash 2.1.0", + "twox-hash 2.1.1", "url", ] @@ -8156,7 +8011,7 @@ dependencies = [ "quote", "serde", "serde_json", - "syn 2.0.101", + "syn 2.0.103", ] [[package]] @@ -8351,7 +8206,7 @@ dependencies = [ "proc-macro-error", "proc-macro2", "quote", - "syn 2.0.101", + "syn 2.0.103", ] [[package]] @@ -8619,7 +8474,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.101", + "syn 2.0.103", ] [[package]] @@ -8639,7 +8494,7 @@ dependencies = [ "getrandom 0.2.16", "http 1.3.1", "rand 0.8.5", - "reqwest 0.12.18", + "reqwest 0.12.20", "serde", "serde_json", "serde_path_to_error", @@ -8688,7 +8543,7 @@ dependencies = [ "percent-encoding", "quick-xml 0.37.5", "rand 0.9.0", - "reqwest 0.12.18", + "reqwest 0.12.20", "ring 0.17.14", "serde", "serde_json", @@ -8837,7 +8692,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.101", + "syn 2.0.103", ] [[package]] @@ -9152,7 +9007,7 @@ checksum = "bc838d2a56b5b1a6c25f55575dfc605fabb63bb2365f6c2353ef9159aa69e4a5" dependencies = [ "cfg-if", "libc", - "redox_syscall 0.5.12", + "redox_syscall 0.5.13", "smallvec", "windows-targets 0.52.6", ] @@ -9164,13 +9019,13 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "be7b2d778f6b841d37083ebdf32e33a524acde1266b5884a8ca29bf00dfa1231" dependencies = [ "ahash 0.8.12", - "arrow-array 55.1.0", - "arrow-buffer 55.1.0", - "arrow-cast 55.1.0", - "arrow-data 55.1.0", + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-data", "arrow-ipc", - "arrow-schema 55.1.0", - "arrow-select 55.1.0", + "arrow-schema", + "arrow-select", "base64 0.22.1", "brotli 8.0.1", "bytes", @@ -9178,7 +9033,7 @@ dependencies = [ "flate2", "futures", "half", - "hashbrown 0.15.3", + "hashbrown 0.15.4", "lz4_flex", "num", "num-bigint", @@ -9189,7 +9044,7 @@ dependencies = [ "snap", "thrift", "tokio", - "twox-hash 2.1.0", + "twox-hash 2.1.1", "zstd", ] @@ -9277,7 +9132,7 @@ checksum = "31095ca1f396e3de32745f42b20deef7bc09077f918b085307e8eab6ddd8fb9c" dependencies = [ "once_cell", "serde", - "unicode-width 0.2.0", + "unicode-width 0.2.1", "unscanny", ] @@ -9346,7 +9201,7 @@ dependencies = [ "phf_shared", "proc-macro2", "quote", - "syn 2.0.101", + "syn 2.0.103", ] [[package]] @@ -9365,7 +9220,7 @@ source = "git+https://github.com/php-rust-tools/parser?rev=ec4cb411dec09450946ef dependencies = [ "ariadne", "clap", - "schemars", + "schemars 0.8.22", "serde", "serde_json", ] @@ -9387,7 +9242,7 @@ checksum = "6e918e4ff8c4549eb882f14b3a4bc8c8bc93de829416eacf579f1207a8fbf861" dependencies = [ "proc-macro2", "quote", - "syn 2.0.101", + "syn 2.0.103", ] [[package]] @@ -9456,7 +9311,7 @@ dependencies = [ "crc32fast", "fdeflate", "flate2", - "miniz_oxide 0.8.8", + "miniz_oxide 0.8.9", ] [[package]] @@ -9473,9 +9328,9 @@ dependencies = [ [[package]] name = "portable-atomic" -version = "1.11.0" +version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "350e9b48cbc6b0e028b0473b114454c6316e57336ee184ceab6e53f72c178b3e" +checksum = "f84267b20a16ea918e43c6a88433c2d54fa145c92a811b5b047ccbe153674483" [[package]] name = "postgres-native-tls" @@ -9594,12 +9449,12 @@ checksum = "c6fa0831dd7cc608c38a5e323422a0077678fa5744aa2be4ad91c4ece8eec8d5" [[package]] name = "prettyplease" -version = "0.2.33" +version = "0.2.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9dee91521343f4c5c6a63edd65e54f31f5c92fe8978c40a4282f8372194c6a7d" +checksum = "6837b9e10d61f45f987d50808f83d1ee3d206c66acf650c3e4ae2e1f6ddedf55" dependencies = [ "proc-macro2", - "syn 2.0.101", + "syn 2.0.103", ] [[package]] @@ -9617,7 +9472,7 @@ version = "3.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "edce586971a4dfaa28950c6f18ed55e0406c1ab88bbce2c6f6293a7aaba73d35" dependencies = [ - "toml_edit 0.22.26", + "toml_edit 0.22.27", ] [[package]] @@ -9663,7 +9518,7 @@ dependencies = [ "proc-macro-error-attr2", "proc-macro2", "quote", - "syn 2.0.101", + "syn 2.0.103", ] [[package]] @@ -9674,7 +9529,7 @@ checksum = "07c277e4e643ef00c1233393c673f655e3672cf7eb3ba08a00bdd0ea59139b5f" dependencies = [ "proc-macro-rules-macros", "proc-macro2", - "syn 2.0.101", + "syn 2.0.103", ] [[package]] @@ -9686,7 +9541,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.101", + "syn 2.0.103", ] [[package]] @@ -9783,7 +9638,7 @@ dependencies = [ "prost", "prost-types", "regex", - "syn 2.0.101", + "syn 2.0.103", "tempfile", ] @@ -9797,7 +9652,7 @@ dependencies = [ "itertools 0.14.0", "proc-macro2", "quote", - "syn 2.0.101", + "syn 2.0.103", ] [[package]] @@ -9920,7 +9775,7 @@ checksum = "6b450dad8382b1b95061d5ca1eb792081fb082adf48c678791fe917509596d5f" dependencies = [ "ahash 0.8.12", "equivalent", - "hashbrown 0.15.3", + "hashbrown 0.15.4", "parking_lot 0.12.4", ] @@ -9936,7 +9791,7 @@ dependencies = [ "quinn-proto", "quinn-udp", "rustc-hash 2.1.1", - "rustls 0.23.27", + "rustls 0.23.28", "socket2", "thiserror 2.0.12", "tokio", @@ -9956,7 +9811,7 @@ dependencies = [ "rand 0.9.0", "ring 0.17.14", "rustc-hash 2.1.1", - "rustls 0.23.27", + "rustls 0.23.28", "rustls-pki-types", "slab", "thiserror 2.0.12", @@ -9990,9 +9845,9 @@ dependencies = [ [[package]] name = "r-efi" -version = "5.2.0" +version = "5.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74765f6d916ee2faa39bc8e68e4f3ed8949b48cccdac59983d287a7cb71ce9c5" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" [[package]] name = "radium" @@ -10206,7 +10061,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "76009fbe0614077fc1a2ce255e3a1881a2e3a3527097d5dc6d8212c585e7e38b" dependencies = [ "quote", - "syn 2.0.101", + "syn 2.0.103", ] [[package]] @@ -10229,9 +10084,9 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.5.12" +version = "0.5.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "928fca9cf2aa042393a8325b9ead81d2f0df4cb12e1e24cef072922ccd99c5af" +checksum = "0d04b7d0ee6b4a0207a0a7adb104d23ecb0b47d6beae7152d0fa34b692b29fd6" dependencies = [ "bitflags 2.9.1", ] @@ -10264,7 +10119,7 @@ checksum = "1165225c21bff1f3bbce98f5a1f889949bc902d3575308cc7b0de30b4f6d27c7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.101", + "syn 2.0.103", ] [[package]] @@ -10376,9 +10231,9 @@ dependencies = [ [[package]] name = "reqwest" -version = "0.12.18" +version = "0.12.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e98ff6b0dbbe4d5a37318f433d4fc82babd21631f194d370409ceb2e40b2f0b5" +checksum = "eabf4c97d9130e2bf606614eb937e86edac8292eaa6f422f995d7e8de1eb1813" dependencies = [ "async-compression", "base64 0.22.1", @@ -10391,19 +10246,17 @@ dependencies = [ "http-body 1.0.1", "http-body-util", "hyper 1.6.0", - "hyper-rustls 0.27.6", + "hyper-rustls 0.27.7", "hyper-tls 0.6.0", "hyper-util", - "ipnet", "js-sys", "log", "mime", "native-tls", - "once_cell", "percent-encoding", "pin-project-lite", "quinn", - "rustls 0.23.27", + "rustls 0.23.28", "rustls-native-certs 0.8.1", "rustls-pki-types", "serde", @@ -10434,7 +10287,7 @@ dependencies = [ "anyhow", "async-trait", "http 1.3.1", - "reqwest 0.12.18", + "reqwest 0.12.20", "serde", "thiserror 1.0.69", "tower-service", @@ -10453,7 +10306,7 @@ dependencies = [ "http 1.3.1", "hyper 1.6.0", "parking_lot 0.11.2", - "reqwest 0.12.18", + "reqwest 0.12.20", "reqwest-middleware", "retry-policies", "thiserror 1.0.69", @@ -10557,35 +10410,41 @@ dependencies = [ [[package]] name = "rmcp" version = "0.1.5" -source = "git+https://github.com/windmill-labs/rust-sdk#9142b40202e49ca0b6530fa49a3abc8bd1b2fcf0" +source = "git+https://github.com/modelcontextprotocol/rust-sdk#d1884fe598cd4c524ff6e0101859a60d4b4c8b8f" dependencies = [ - "async-stream", - "axum", - "base64 0.21.7", + "base64 0.22.1", + "bytes", "chrono", "futures", + "http 1.3.1", + "http-body 1.0.1", + "http-body-util", "paste", "pin-project-lite", "rand 0.9.0", "rmcp-macros", - "schemars", + "schemars 0.8.22", "serde", "serde_json", + "sse-stream", "thiserror 2.0.12", "tokio", "tokio-stream", "tokio-util", + "tower-service", "tracing", + "uuid", ] [[package]] name = "rmcp-macros" version = "0.1.5" -source = "git+https://github.com/windmill-labs/rust-sdk#9142b40202e49ca0b6530fa49a3abc8bd1b2fcf0" +source = "git+https://github.com/modelcontextprotocol/rust-sdk#d1884fe598cd4c524ff6e0101859a60d4b4c8b8f" dependencies = [ "proc-macro2", "quote", - "syn 2.0.101", + "serde_json", + "syn 2.0.103", ] [[package]] @@ -10675,7 +10534,7 @@ dependencies = [ "quote", "rust-embed-utils", "shellexpand", - "syn 2.0.101", + "syn 2.0.103", "walkdir", ] @@ -10701,9 +10560,9 @@ dependencies = [ [[package]] name = "rust_decimal" -version = "1.37.1" +version = "1.37.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "faa7de2ba56ac291bd90c6b9bece784a52ae1411f9506544b3eae36dd2356d50" +checksum = "b203a6425500a03e0919c42d3c47caca51e79f1132046626d2c8871c5092035d" dependencies = [ "arrayvec", "borsh", @@ -10718,9 +10577,9 @@ dependencies = [ [[package]] name = "rustc-demangle" -version = "0.1.24" +version = "0.1.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f" +checksum = "989e6739f80c4ad5b13e0fd7fe89531180375b18520cc8c82080e4dc4035b84f" [[package]] name = "rustc-hash" @@ -10815,9 +10674,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.27" +version = "0.23.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "730944ca083c1c233a75c09f199e973ca499344a2b7ba9e755c457e86fb4a321" +checksum = "7160e3e10bf4535308537f3c4e1641468cd0e485175d6163087c0393c7d46643" dependencies = [ "aws-lc-rs", "log", @@ -10901,7 +10760,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "22557157d7395bc30727745b365d923f1ecc230c4c80b176545f3f4f08c46e33" dependencies = [ "futures", - "rustls 0.23.27", + "rustls 0.23.28", "socket2", "tokio", ] @@ -11122,12 +10981,25 @@ version = "0.8.22" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3fbf2ae1b8bc8e02df939598064d22402220cd5bbcca1c76f7d6a310974d5615" dependencies = [ + "chrono", "dyn-clone", "schemars_derive", "serde", "serde_json", ] +[[package]] +name = "schemars" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cd191f9397d57d581cddd31014772520aa448f65ef991055d7f61582c65165f" +dependencies = [ + "dyn-clone", + "ref-cast", + "serde", + "serde_json", +] + [[package]] name = "schemars_derive" version = "0.8.22" @@ -11137,7 +11009,7 @@ dependencies = [ "proc-macro2", "quote", "serde_derive_internals", - "syn 2.0.101", + "syn 2.0.103", ] [[package]] @@ -11317,7 +11189,7 @@ checksum = "5b0276cf7f2c73365f7157c8123c21cd9a50fbbd844757af28ca1f5925fc2a00" dependencies = [ "proc-macro2", "quote", - "syn 2.0.101", + "syn 2.0.103", ] [[package]] @@ -11328,7 +11200,7 @@ checksum = "18d26a20a969b9e3fdf2fc2d9f21eda6c40e2de84c9408bb5d3b05d499aae711" dependencies = [ "proc-macro2", "quote", - "syn 2.0.101", + "syn 2.0.103", ] [[package]] @@ -11380,14 +11252,14 @@ checksum = "175ee3e80ae9982737ca543e96133087cbd9a485eecc3bc4de9c1a37b47ea59c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.101", + "syn 2.0.103", ] [[package]] name = "serde_spanned" -version = "0.6.8" +version = "0.6.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87607cb1398ed59d48732e575a4c28a7a8ebf2454b964fe3f224f2afc07909e1" +checksum = "bf41e0cfaf7226dca15e8197172c295a782857fcb97fad1808a166870dee75a3" dependencies = [ "serde", ] @@ -11420,15 +11292,16 @@ dependencies = [ [[package]] name = "serde_with" -version = "3.12.0" +version = "3.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6b6f7f2fcb69f747921f79f3926bd1e203fce4fef62c268dd3abfb6d86029aa" +checksum = "bf65a400f8f66fb7b0552869ad70157166676db75ed8181f8104ea91cf9d0b42" dependencies = [ "base64 0.22.1", "chrono", "hex", "indexmap 1.9.3", "indexmap 2.9.0", + "schemars 0.9.0", "serde", "serde_derive", "serde_json", @@ -11438,14 +11311,29 @@ dependencies = [ [[package]] name = "serde_with_macros" -version = "3.12.0" +version = "3.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d00caa5193a3c8362ac2b73be6b9e768aa5a4b2f721d8f4b339600c3cb51f8e" +checksum = "81679d9ed988d5e9a5e6531dc3f2c28efbd639cbd1dfb628df08edea6004da77" dependencies = [ "darling 0.20.11", "proc-macro2", "quote", - "syn 2.0.101", + "syn 2.0.103", +] + +[[package]] +name = "serde_yml" +version = "0.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59e2dd588bf1597a252c3b920e0143eb99b0f76e4e082f4c92ce34fbc9e71ddd" +dependencies = [ + "indexmap 2.9.0", + "itoa", + "libyml", + "memchr", + "ryu", + "serde", + "version_check", ] [[package]] @@ -11645,12 +11533,9 @@ dependencies = [ [[package]] name = "slab" -version = "0.4.9" +version = "0.4.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f92a496fb766b417c996b9c5e57daf2f7ad3b0bebe1ccfca4856390e3d3bb67" -dependencies = [ - "autocfg", -] +checksum = "04dc19736151f35336d325007ac991178d504a119863a2fcb3758cdb5e52c50d" [[package]] name = "slotmap" @@ -11672,9 +11557,9 @@ dependencies = [ [[package]] name = "smallvec" -version = "1.15.0" +version = "1.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8917285742e9f3e1683f0a9c4e6b57960b7314d0b08d30d1ecd426713ee2eee9" +checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" dependencies = [ "serde", ] @@ -11733,9 +11618,9 @@ dependencies = [ [[package]] name = "sourcemap" -version = "9.2.1" +version = "9.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bdee719193ae5c919a3ee43f64c2c0dd87f9b9a451d67918a2a5ec2e3c70561c" +checksum = "e22afbcb92ce02d23815b9795523c005cb9d3c214f8b7a66318541c240ea7935" dependencies = [ "base64-simd 0.8.0", "bitvec", @@ -11830,7 +11715,7 @@ checksum = "da5fc6819faabb412da764b99d3b713bb55083c11e7e0c00144d386cd6a1939c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.101", + "syn 2.0.103", ] [[package]] @@ -11864,14 +11749,14 @@ dependencies = [ "futures-intrusive", "futures-io", "futures-util", - "hashbrown 0.15.3", + "hashbrown 0.15.4", "hashlink 0.10.0", "indexmap 2.9.0", "log", "memchr", "once_cell", "percent-encoding", - "rustls 0.23.27", + "rustls 0.23.28", "serde", "serde_json", "sha2 0.10.9", @@ -11895,7 +11780,7 @@ dependencies = [ "quote", "sqlx-core", "sqlx-macros-core", - "syn 2.0.101", + "syn 2.0.103", ] [[package]] @@ -11918,7 +11803,7 @@ dependencies = [ "sqlx-mysql", "sqlx-postgres", "sqlx-sqlite", - "syn 2.0.101", + "syn 2.0.103", "tokio", "url", ] @@ -12035,6 +11920,19 @@ dependencies = [ "uuid", ] +[[package]] +name = "sse-stream" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f649a9f9e91db2ed32f3724516eac2bc09fab77fc33be8f670f5619b9dc6c3f" +dependencies = [ + "bytes", + "futures-util", + "http-body 1.0.1", + "http-body-util", + "pin-project-lite", +] + [[package]] name = "stable_deref_trait" version = "1.2.0" @@ -12069,7 +11967,7 @@ dependencies = [ "proc-macro2", "quote", "swc_macros_common", - "syn 2.0.101", + "syn 2.0.103", ] [[package]] @@ -12138,7 +12036,7 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.101", + "syn 2.0.103", ] [[package]] @@ -12151,7 +12049,7 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.101", + "syn 2.0.103", ] [[package]] @@ -12237,7 +12135,7 @@ dependencies = [ "rustc-hash 1.1.0", "serde", "siphasher 0.3.11", - "sourcemap 9.2.1", + "sourcemap 9.2.2", "swc_allocator", "swc_atoms", "swc_eq_ignore_macros", @@ -12270,7 +12168,7 @@ dependencies = [ "proc-macro2", "quote", "swc_macros_common", - "syn 2.0.101", + "syn 2.0.103", ] [[package]] @@ -12301,7 +12199,7 @@ dependencies = [ "num-bigint", "once_cell", "serde", - "sourcemap 9.2.1", + "sourcemap 9.2.2", "swc_allocator", "swc_atoms", "swc_common", @@ -12319,7 +12217,7 @@ dependencies = [ "proc-macro2", "quote", "swc_macros_common", - "syn 2.0.101", + "syn 2.0.103", ] [[package]] @@ -12404,7 +12302,7 @@ dependencies = [ "proc-macro2", "quote", "swc_macros_common", - "syn 2.0.101", + "syn 2.0.103", ] [[package]] @@ -12511,7 +12409,7 @@ checksum = "63db0adcff29d220c3d151c5b25c0eabe7e32dd936212b84cdaa1392e3130497" dependencies = [ "proc-macro2", "quote", - "syn 2.0.101", + "syn 2.0.103", ] [[package]] @@ -12522,7 +12420,7 @@ checksum = "f486687bfb7b5c560868f69ed2d458b880cebc9babebcb67e49f31b55c5bf847" dependencies = [ "proc-macro2", "quote", - "syn 2.0.101", + "syn 2.0.103", ] [[package]] @@ -12545,7 +12443,7 @@ dependencies = [ "proc-macro2", "quote", "swc_macros_common", - "syn 2.0.101", + "syn 2.0.103", ] [[package]] @@ -12561,9 +12459,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.101" +version = "2.0.103" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ce2b7fc941b3a24138a0a7cf8e858bfc6a992e7978a068a5c760deb0ed43caf" +checksum = "e4307e30089d6fd6aff212f2da3a1f9e32f3223b1f010fb09b7c95f90f3ca1e8" dependencies = [ "proc-macro2", "quote", @@ -12605,7 +12503,7 @@ checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.101", + "syn 2.0.103", ] [[package]] @@ -12937,7 +12835,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c13547615a44dc9c452a8a534638acdf07120d4b6847c8178705da06306a3057" dependencies = [ "unicode-linebreak", - "unicode-width 0.2.0", + "unicode-width 0.2.1", ] [[package]] @@ -12966,7 +12864,7 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.101", + "syn 2.0.103", ] [[package]] @@ -12977,17 +12875,16 @@ checksum = "7f7cf42b4507d8ea322120659672cf1b9dbb93f8f2d4ecfd6e51350ff5b17a1d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.101", + "syn 2.0.103", ] [[package]] name = "thread_local" -version = "1.1.8" +version = "1.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b9ef9bad013ada3808854ceac7b46812a6465ba368859a37e2100283d2d719c" +checksum = "f60246a4944f24f6e018aa17cdeffb7818b76356965d03b07d6a9886e8962185" dependencies = [ "cfg-if", - "once_cell", ] [[package]] @@ -13146,7 +13043,7 @@ dependencies = [ "bincode", "lazy_static", "rayon", - "schemars", + "schemars 0.8.22", "serde", "thiserror 1.0.69", "tracing", @@ -13222,7 +13119,7 @@ checksum = "6e06d43f1345a3bcd39f6a56dbb7dcab2ba47e68e8ac134855e7e2bdbaf8cab8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.101", + "syn 2.0.103", ] [[package]] @@ -13335,7 +13232,7 @@ version = "0.26.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e727b36a1a0e8b74c376ac2211e40c2c8af09fb4013c60d910495810f008e9b" dependencies = [ - "rustls 0.23.27", + "rustls 0.23.28", "tokio", ] @@ -13402,7 +13299,7 @@ dependencies = [ "futures-io", "futures-sink", "futures-util", - "hashbrown 0.15.3", + "hashbrown 0.15.4", "pin-project-lite", "slab", "tokio", @@ -13443,9 +13340,9 @@ dependencies = [ [[package]] name = "toml_datetime" -version = "0.6.9" +version = "0.6.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3da5db5a963e24bc68be8b17b6fa82814bb22ee8660f192bb182771d498f09a3" +checksum = "22cddaf88f4fbc13c51aebbf5f8eceb5c7c5a9da2ac40a13519eb5b0a0e8f11c" dependencies = [ "serde", ] @@ -13465,13 +13362,13 @@ dependencies = [ [[package]] name = "toml_edit" -version = "0.22.26" +version = "0.22.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "310068873db2c5b3e7659d2cc35d21855dbafa50d1ce336397c666e3cb08137e" +checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a" dependencies = [ "indexmap 2.9.0", "toml_datetime", - "winnow 0.7.10", + "winnow 0.7.11", ] [[package]] @@ -13564,9 +13461,9 @@ dependencies = [ [[package]] name = "tower-http" -version = "0.6.4" +version = "0.6.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fdb0c213ca27a9f57ab69ddb290fd80d970922355b83ae380b395d3986b8a2e" +checksum = "adc82fd73de2a9722ac5da747f12383d2bfdb93591ee6c58486e0097890f05f2" dependencies = [ "async-compression", "bitflags 2.9.1", @@ -13624,20 +13521,20 @@ dependencies = [ [[package]] name = "tracing-attributes" -version = "0.1.28" +version = "0.1.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "395ae124c09f9e6918a2310af6038fba074bcf474ac352496d5910dd59a2226d" +checksum = "81383ab64e72a7a8b8e13130c49e3dab29def6d0c7d76a03087b3cf71c5c6903" dependencies = [ "proc-macro2", "quote", - "syn 2.0.101", + "syn 2.0.103", ] [[package]] name = "tracing-core" -version = "0.1.33" +version = "0.1.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e672c95779cf947c5311f83787af4fa8fffd12fb27e4993211a84bdfd9610f9c" +checksum = "b9d12581f227e93f094d3af2ae690a574abb8a2b9b7a96e7cfe9647b2b617678" dependencies = [ "once_cell", "valuable", @@ -13671,7 +13568,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ba3beec919fbdf99d719de8eda6adae3281f8a5b71ae40431f44dc7423053d34" dependencies = [ "loki-api", - "reqwest 0.12.18", + "reqwest 0.12.20", "serde", "serde_json", "snap", @@ -13831,9 +13728,9 @@ dependencies = [ [[package]] name = "twox-hash" -version = "2.1.0" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7b17f197b3050ba473acf9181f7b1d3b66d1cf7356c6cc57886662276e65908" +checksum = "8b907da542cbced5261bd3256de1b3a1bf340a3d37f93425a07362a1d687de56" [[package]] name = "typed-arena" @@ -13874,7 +13771,7 @@ checksum = "35f5380909ffc31b4de4f4bdf96b877175a016aa2ca98cee39fcfd8c4d53d952" dependencies = [ "proc-macro2", "quote", - "syn 2.0.101", + "syn 2.0.103", ] [[package]] @@ -14047,9 +13944,9 @@ checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af" [[package]] name = "unicode-width" -version = "0.2.0" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fc81956842c57dac11422a97c3b8195a1ff727f06e85c84ed2e8aa277c9a0fd" +checksum = "4a1a07cc7db3810833284e8d372ccdc6da29741639ecc70c9ec107df0fa6154c" [[package]] name = "unicode-xid" @@ -14124,7 +14021,7 @@ dependencies = [ "log", "native-tls", "once_cell", - "rustls 0.23.27", + "rustls 0.23.28", "rustls-pki-types", "serde", "serde_json", @@ -14302,9 +14199,9 @@ dependencies = [ [[package]] name = "wasi" -version = "0.11.0+wasi-snapshot-preview1" +version = "0.11.1+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" [[package]] name = "wasi" @@ -14343,7 +14240,7 @@ dependencies = [ "log", "proc-macro2", "quote", - "syn 2.0.101", + "syn 2.0.103", "wasm-bindgen-shared", ] @@ -14378,7 +14275,7 @@ checksum = "8ae87ea40c9f689fc23f209965b6fb8a99ad69aeeb0231408be24920604395de" dependencies = [ "proc-macro2", "quote", - "syn 2.0.101", + "syn 2.0.103", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -14413,7 +14310,7 @@ checksum = "17d5042cc5fa009658f9a7333ef24291b1291a25b6382dd68862a7f3b969f69b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.101", + "syn 2.0.103", ] [[package]] @@ -14623,7 +14520,7 @@ version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6994d13118ab492c3c80c1f81928718159254c53c472bf9ce36f8dae4add02a7" dependencies = [ - "redox_syscall 0.5.12", + "redox_syscall 0.5.13", "wasite", "web-sys", ] @@ -14667,7 +14564,7 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "windmill" -version = "1.494.0" +version = "1.498.0" dependencies = [ "anyhow", "axum", @@ -14687,10 +14584,11 @@ dependencies = [ "prometheus", "quote", "rand 0.9.0", - "reqwest 0.12.18", - "rustls 0.23.27", + "reqwest 0.12.20", + "rustls 0.23.28", "serde", "serde_json", + "serde_yml", "sha1", "sha2 0.10.9", "size", @@ -14718,7 +14616,7 @@ dependencies = [ [[package]] name = "windmill-api" -version = "1.494.0" +version = "1.498.0" dependencies = [ "anyhow", "argon2", @@ -14756,6 +14654,7 @@ dependencies = [ "hmac", "http 1.3.1", "hyper 1.6.0", + "indexmap 2.9.0", "itertools 0.14.0", "jsonwebtoken 8.3.0", "lazy_static", @@ -14777,7 +14676,7 @@ dependencies = [ "rand 0.9.0", "rdkafka", "regex", - "reqwest 0.12.18", + "reqwest 0.12.20", "rmcp", "rsa", "rumqttc", @@ -14787,6 +14686,7 @@ dependencies = [ "serde", "serde_json", "serde_urlencoded", + "serde_yml", "sha1", "sha2 0.10.9", "sql-builder", @@ -14827,7 +14727,7 @@ dependencies = [ [[package]] name = "windmill-api-client" -version = "1.494.0" +version = "1.498.0" dependencies = [ "base64 0.22.1", "chrono", @@ -14842,7 +14742,7 @@ dependencies = [ [[package]] name = "windmill-audit" -version = "1.494.0" +version = "1.498.0" dependencies = [ "chrono", "serde", @@ -14855,7 +14755,7 @@ dependencies = [ [[package]] name = "windmill-autoscaling" -version = "1.494.0" +version = "1.498.0" dependencies = [ "anyhow", "serde", @@ -14869,7 +14769,7 @@ dependencies = [ [[package]] name = "windmill-common" -version = "1.494.0" +version = "1.498.0" dependencies = [ "anyhow", "async-stream", @@ -14911,7 +14811,7 @@ dependencies = [ "quick_cache", "rand 0.9.0", "regex", - "reqwest 0.12.18", + "reqwest 0.12.20", "reqwest-middleware", "reqwest-retry", "semver 1.0.26", @@ -14936,6 +14836,7 @@ dependencies = [ "tracing-loki", "tracing-opentelemetry", "tracing-subscriber", + "url", "uuid", "windmill-macros", "windmill-parser-py", @@ -14945,7 +14846,7 @@ dependencies = [ [[package]] name = "windmill-git-sync" -version = "1.494.0" +version = "1.498.0" dependencies = [ "regex", "serde", @@ -14959,7 +14860,7 @@ dependencies = [ [[package]] name = "windmill-indexer" -version = "1.494.0" +version = "1.498.0" dependencies = [ "anyhow", "bytes", @@ -14982,19 +14883,19 @@ dependencies = [ [[package]] name = "windmill-macros" -version = "1.494.0" +version = "1.498.0" dependencies = [ "itertools 0.14.0", "lazy_static", "proc-macro2", "quote", "regex", - "syn 2.0.101", + "syn 2.0.103", ] [[package]] name = "windmill-parser" -version = "1.494.0" +version = "1.498.0" dependencies = [ "convert_case 0.6.0", "serde", @@ -15003,7 +14904,7 @@ dependencies = [ [[package]] name = "windmill-parser-bash" -version = "1.494.0" +version = "1.498.0" dependencies = [ "anyhow", "lazy_static", @@ -15015,7 +14916,7 @@ dependencies = [ [[package]] name = "windmill-parser-csharp" -version = "1.494.0" +version = "1.498.0" dependencies = [ "anyhow", "serde_json", @@ -15027,7 +14928,7 @@ dependencies = [ [[package]] name = "windmill-parser-go" -version = "1.494.0" +version = "1.498.0" dependencies = [ "anyhow", "gosyn", @@ -15039,7 +14940,7 @@ dependencies = [ [[package]] name = "windmill-parser-graphql" -version = "1.494.0" +version = "1.498.0" dependencies = [ "anyhow", "lazy_static", @@ -15051,7 +14952,7 @@ dependencies = [ [[package]] name = "windmill-parser-java" -version = "1.494.0" +version = "1.498.0" dependencies = [ "anyhow", "serde_json", @@ -15063,7 +14964,7 @@ dependencies = [ [[package]] name = "windmill-parser-nu" -version = "1.494.0" +version = "1.498.0" dependencies = [ "anyhow", "nu-parser", @@ -15074,7 +14975,7 @@ dependencies = [ [[package]] name = "windmill-parser-php" -version = "1.494.0" +version = "1.498.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -15085,7 +14986,7 @@ dependencies = [ [[package]] name = "windmill-parser-py" -version = "1.494.0" +version = "1.498.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -15096,7 +14997,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-imports" -version = "1.494.0" +version = "1.498.0" dependencies = [ "anyhow", "async-recursion", @@ -15119,7 +15020,7 @@ dependencies = [ [[package]] name = "windmill-parser-rust" -version = "1.494.0" +version = "1.498.0" dependencies = [ "anyhow", "convert_case 0.6.0", @@ -15129,14 +15030,14 @@ dependencies = [ "quote", "regex", "serde_json", - "syn 2.0.101", + "syn 2.0.103", "toml", "windmill-parser", ] [[package]] name = "windmill-parser-sql" -version = "1.494.0" +version = "1.498.0" dependencies = [ "anyhow", "lazy_static", @@ -15148,7 +15049,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts" -version = "1.494.0" +version = "1.498.0" dependencies = [ "anyhow", "lazy_static", @@ -15166,7 +15067,7 @@ dependencies = [ [[package]] name = "windmill-parser-wasm" -version = "1.494.0" +version = "1.498.0" dependencies = [ "anyhow", "getrandom 0.2.16", @@ -15190,7 +15091,7 @@ dependencies = [ [[package]] name = "windmill-parser-yaml" -version = "1.494.0" +version = "1.498.0" dependencies = [ "anyhow", "serde_json", @@ -15200,7 +15101,7 @@ dependencies = [ [[package]] name = "windmill-queue" -version = "1.494.0" +version = "1.498.0" dependencies = [ "anyhow", "async-recursion", @@ -15217,7 +15118,7 @@ dependencies = [ "lazy_static", "prometheus", "regex", - "reqwest 0.12.18", + "reqwest 0.12.20", "serde", "serde_json", "serde_urlencoded", @@ -15233,7 +15134,7 @@ dependencies = [ [[package]] name = "windmill-sql-datatype-parser-wasm" -version = "1.494.0" +version = "1.498.0" dependencies = [ "wasm-bindgen", "wasm-bindgen-test", @@ -15243,7 +15144,7 @@ dependencies = [ [[package]] name = "windmill-worker" -version = "1.494.0" +version = "1.498.0" dependencies = [ "anyhow", "async-recursion", @@ -15295,7 +15196,7 @@ dependencies = [ "prometheus", "rand 0.9.0", "regex", - "reqwest 0.12.18", + "reqwest 0.12.20", "reqwest-middleware", "rust_decimal", "serde", @@ -15357,9 +15258,9 @@ dependencies = [ [[package]] name = "windows" -version = "0.61.1" +version = "0.61.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c5ee8f3d025738cb02bad7868bbb5f8a6327501e870bf51f1b455b0a2454a419" +checksum = "9babd3a767a4c1aef6900409f85f5d53ce2544ccdfaa86dad48c91782c6d6893" dependencies = [ "windows-collections", "windows-core 0.61.2", @@ -15411,7 +15312,7 @@ dependencies = [ "windows-interface 0.59.1", "windows-link", "windows-result 0.3.4", - "windows-strings 0.4.2", + "windows-strings", ] [[package]] @@ -15433,7 +15334,7 @@ checksum = "f6fc35f58ecd95a9b71c4f2329b911016e6bec66b3f2e6a4aad86bd2e99e2f9b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.101", + "syn 2.0.103", ] [[package]] @@ -15444,7 +15345,7 @@ checksum = "9107ddc059d5b6fbfbffdfa7a7fe3e22a226def0b2608f72e9d552763d3e1ad7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.101", + "syn 2.0.103", ] [[package]] @@ -15455,7 +15356,7 @@ checksum = "a47fddd13af08290e67f4acabf4b459f647552718f683a7b415d290ac744a836" dependencies = [ "proc-macro2", "quote", - "syn 2.0.101", + "syn 2.0.103", ] [[package]] @@ -15466,7 +15367,7 @@ checksum = "08990546bf4edef8f431fa6326e032865f27138718c587dc21bc0265bbcb57cc" dependencies = [ "proc-macro2", "quote", - "syn 2.0.101", + "syn 2.0.103", ] [[package]] @@ -15477,7 +15378,7 @@ checksum = "29bee4b38ea3cde66011baa44dba677c432a78593e202392d1e9070cf2a7fca7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.101", + "syn 2.0.103", ] [[package]] @@ -15488,14 +15389,14 @@ checksum = "bd9211b69f8dcdfa817bfd14bf1c97c9188afa36f4750130fcdf3f400eca9fa8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.101", + "syn 2.0.103", ] [[package]] name = "windows-link" -version = "0.1.1" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76840935b766e1b0a05c0066835fb9ec80071d4c09a16f6bd5f7e655e3c14c38" +checksum = "5e6ad25900d524eaabdbbb96d20b4311e1e7ae1699af4fb28c17ae66c80d798a" [[package]] name = "windows-numerics" @@ -15509,13 +15410,13 @@ dependencies = [ [[package]] name = "windows-registry" -version = "0.4.0" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4286ad90ddb45071efd1a66dfa43eb02dd0dfbae1545ad6cc3c51cf34d7e8ba3" +checksum = "b3bab093bdd303a1240bb99b8aba8ea8a69ee19d34c9e2ef9594e708a4878820" dependencies = [ + "windows-link", "windows-result 0.3.4", - "windows-strings 0.3.1", - "windows-targets 0.53.0", + "windows-strings", ] [[package]] @@ -15536,15 +15437,6 @@ dependencies = [ "windows-link", ] -[[package]] -name = "windows-strings" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87fa48cc5d406560701792be122a10132491cff9d0aeb23583cc2dcafc847319" -dependencies = [ - "windows-link", -] - [[package]] name = "windows-strings" version = "0.4.2" @@ -15614,9 +15506,9 @@ dependencies = [ [[package]] name = "windows-targets" -version = "0.53.0" +version = "0.53.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1e4c7e8ceaaf9cb7d7507c974735728ab453b67ef8f18febdd7c11fe59dca8b" +checksum = "c66f69fcc9ce11da9966ddb31a40968cad001c5bedeb5c2b82ede4253ab48aef" dependencies = [ "windows_aarch64_gnullvm 0.53.0", "windows_aarch64_msvc 0.53.0", @@ -15795,9 +15687,9 @@ dependencies = [ [[package]] name = "winnow" -version = "0.7.10" +version = "0.7.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c06928c8748d81b05c9be96aad92e1b6ff01833332f281e8cfca3be4b35fc9ec" +checksum = "74c7b26e3480b707944fc872477815d29a8e429d2f93a1ce000f5fa84a15cbcd" dependencies = [ "memchr", ] @@ -15955,7 +15847,7 @@ checksum = "2380878cad4ac9aac1e2435f3eb4020e8374b5f13c296cb75b4620ff8e229154" dependencies = [ "proc-macro2", "quote", - "syn 2.0.101", + "syn 2.0.103", "synstructure 0.13.2", ] @@ -15967,7 +15859,7 @@ checksum = "38da3c9736e16c5d3c8c597a9aaa5d1fa565d0532ae05e27c24aa62fb32c0ab6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.101", + "syn 2.0.103", "synstructure 0.13.2", ] @@ -15988,7 +15880,7 @@ checksum = "28a6e20d751156648aa063f3800b706ee209a32c0b4d9f24be3d980b01be55ef" dependencies = [ "proc-macro2", "quote", - "syn 2.0.101", + "syn 2.0.103", ] [[package]] @@ -16008,7 +15900,7 @@ checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" dependencies = [ "proc-macro2", "quote", - "syn 2.0.101", + "syn 2.0.103", "synstructure 0.13.2", ] @@ -16029,7 +15921,7 @@ checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" dependencies = [ "proc-macro2", "quote", - "syn 2.0.101", + "syn 2.0.103", ] [[package]] @@ -16062,7 +15954,7 @@ checksum = "5b96237efa0c878c64bd89c436f661be4e46b2f3eff1ebb976f7ef2321d2f58f" dependencies = [ "proc-macro2", "quote", - "syn 2.0.101", + "syn 2.0.103", ] [[package]] @@ -16082,9 +15974,9 @@ dependencies = [ [[package]] name = "zlib-rs" -version = "0.5.0" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "868b928d7949e09af2f6086dfc1e01936064cc7a819253bce650d4e2a2d63ba8" +checksum = "626bd9fa9734751fc50d6060752170984d7053f5a39061f524cda68023d4db8a" [[package]] name = "zstd" diff --git a/backend/Cargo.toml b/backend/Cargo.toml index 65c1fa0ec3..1ac013b916 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "windmill" -version = "1.494.0" +version = "1.498.0" authors.workspace = true edition.workspace = true @@ -32,7 +32,7 @@ members = [ ] [workspace.package] -version = "1.494.0" +version = "1.498.0" authors = ["Ruben Fiszel "] edition = "2021" @@ -49,6 +49,7 @@ lto = "thin" [features] default = [] +private = ["windmill-api/private", "windmill-autoscaling/private", "windmill-common/private", "windmill-git-sync/private", "windmill-indexer/private", "windmill-queue/private", "windmill-worker/private"] agent_worker_server = ["windmill-api/agent_worker_server"] enterprise = ["windmill-worker/enterprise", "windmill-queue/enterprise", "windmill-api/enterprise", "dep:windmill-autoscaling", "windmill-autoscaling/enterprise", "windmill-git-sync/enterprise", "windmill-common/prometheus", "windmill-common/enterprise"] enterprise_saml = ["windmill-api/enterprise_saml", "oauth2"] @@ -94,8 +95,10 @@ php = ["windmill-worker/php"] csharp = ["windmill-worker/csharp"] nu = ["windmill-worker/nu"] java = ["windmill-worker/java"] -all_languages = [ "python", "deno_core", "rust", "mysql", "oracledb", "duckdb", "mssql", "bigquery", "csharp", "nu", "php", "java"] - +all_languages = ["python", "deno_core", "rust", "mysql", "oracledb", "duckdb", "mssql", "bigquery", "csharp", "nu", "php", "java"] +# For windows we have another set of languages enabled +# NOTE: DuckDB is ignored because of compilation problems +all_languages_windows = ["python", "deno_core", "rust", "mysql", "oracledb", "mssql", "bigquery", "csharp", "nu", "php", "java"] [patch.crates-io] object_store = { git = "https://github.com/apache/arrow-rs-object-store", rev = "36752c975d4f29e20b57c91f81a10872dcd48ae7" } @@ -127,6 +130,7 @@ prometheus = { workspace = true, optional = true } uuid.workspace = true gethostname.workspace = true serde_json.workspace = true +serde_yml.workspace = true serde.workspace = true deno_core = { workspace = true, optional = true } object_store = { workspace = true, optional = true } @@ -198,6 +202,7 @@ tower-http = { version = "^0.6", features = ["trace", "cors"] } tower-cookies = "^0.10" serde = "^1" serde_json = { version = "^1", features = ["preserve_order", "raw_value"] } +serde_yml = "0.0.12" uuid = { version = "^1", features = ["serde", "v4"] } thiserror = "^2" anyhow = "^1" @@ -227,7 +232,7 @@ php-parser-rs = { git = "https://github.com/php-rust-tools/parser", rev = "ec4cb cron = "^0" mail-send = { version = "0.4.0", features = ["builder"], default-features=false } urlencoding = "^2" -url = "^2" +url = { version = "^2" , features = ["serde"]} async-oauth2 = "^0" reqwest = { version = "^0.12", features = ["json", "stream", "gzip"] } time = "^0" diff --git a/backend/all_features_oss.sh b/backend/all_features_oss.sh new file mode 100755 index 0000000000..5871bc656f --- /dev/null +++ b/backend/all_features_oss.sh @@ -0,0 +1,20 @@ +# This script outputs all features except private. Usage : +# > cargo build --features $(./all_features_oss.sh) + +#!/bin/bash + +# Path to the Cargo.toml file +CARGO_TOML_PATH="./Cargo.toml" + +# Extract features from Cargo.toml and output them separated by commas +if [[ -f "$CARGO_TOML_PATH" ]]; then + grep -A 100 '\[features\]' "$CARGO_TOML_PATH" | \ + sed -n '/\[features\]/,/^\[/p' | \ + grep -E '^[a-zA-Z0-9_-]+' | \ + grep -v 'private' | \ + cut -d' ' -f1 | \ + paste -sd ',' - +else + echo "Cargo.toml not found at $CARGO_TOML_PATH" + exit 1 +fi \ No newline at end of file diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index d5d998a701..c41787fda1 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -8a2506e86b923c00522cb83b052586f705f7aa8e +67e727c618cf673850a0887931c803241abfcfe8 \ No newline at end of file diff --git a/backend/migrations/20240917172503_job_routing.down.sql b/backend/migrations/20240917172503_job_routing.down.sql index 569385aa5a..15c3e17762 100644 --- a/backend/migrations/20240917172503_job_routing.down.sql +++ b/backend/migrations/20240917172503_job_routing.down.sql @@ -4,4 +4,4 @@ DROP TYPE http_method; ALTER TABLE script DROP COLUMN has_preprocessor; -DROP FUNCTION prevent_route_path_change(); \ No newline at end of file +DROP FUNCTION prevent_route_path_change(); \ No newline at end of file diff --git a/backend/migrations/20250604181226_auth_cache_invalidation.down.sql b/backend/migrations/20250604181226_auth_cache_invalidation.down.sql new file mode 100644 index 0000000000..b7f0f12204 --- /dev/null +++ b/backend/migrations/20250604181226_auth_cache_invalidation.down.sql @@ -0,0 +1,4 @@ +-- Remove token invalidation notification trigger + +DROP TRIGGER IF EXISTS token_invalidation_trigger ON token; +DROP FUNCTION IF EXISTS notify_token_invalidation(); diff --git a/backend/migrations/20250604181226_auth_cache_invalidation.up.sql b/backend/migrations/20250604181226_auth_cache_invalidation.up.sql new file mode 100644 index 0000000000..11be3961c0 --- /dev/null +++ b/backend/migrations/20250604181226_auth_cache_invalidation.up.sql @@ -0,0 +1,17 @@ +-- Add token invalidation notification trigger + +CREATE OR REPLACE FUNCTION notify_token_invalidation() +RETURNS TRIGGER AS $$ +BEGIN + -- Only notify for session token deletions when the invalidation settings are enabled + IF OLD.label = 'session' AND OLD.email IS NOT NULL THEN + PERFORM pg_notify('notify_token_invalidation', OLD.token); + END IF; + RETURN OLD; +END; +$$ LANGUAGE plpgsql; + +CREATE TRIGGER token_invalidation_trigger +AFTER DELETE ON token +FOR EACH ROW +EXECUTE FUNCTION notify_token_invalidation(); diff --git a/backend/migrations/20250607102516_add_description_and_summary_to_http_trigger.down.sql b/backend/migrations/20250607102516_add_description_and_summary_to_http_trigger.down.sql new file mode 100644 index 0000000000..597a95161d --- /dev/null +++ b/backend/migrations/20250607102516_add_description_and_summary_to_http_trigger.down.sql @@ -0,0 +1,4 @@ +-- Add down migration script here +ALTER TABLE http_trigger +DROP COLUMN summary, +DROP COLUMN description; \ No newline at end of file diff --git a/backend/migrations/20250607102516_add_description_and_summary_to_http_trigger.up.sql b/backend/migrations/20250607102516_add_description_and_summary_to_http_trigger.up.sql new file mode 100644 index 0000000000..4d51f6d07b --- /dev/null +++ b/backend/migrations/20250607102516_add_description_and_summary_to_http_trigger.up.sql @@ -0,0 +1,5 @@ +-- Add up migration script here +ALTER TABLE http_trigger +ADD COLUMN summary VARCHAR(512) NULL, +ADD COLUMN description TEXT NULL; + diff --git a/backend/src/ee.rs b/backend/src/ee_oss.rs similarity index 53% rename from backend/src/ee.rs rename to backend/src/ee_oss.rs index 91816cd1ba..4791a243a3 100644 --- a/backend/src/ee.rs +++ b/backend/src/ee_oss.rs @@ -1,8 +1,13 @@ +#[cfg(feature = "private")] +#[allow(unused)] +pub use crate::ee::*; + +#[cfg(not(feature = "private"))] pub async fn set_license_key(_license_key: String) -> () { // Implementation is not open source } -#[cfg(feature = "enterprise")] +#[cfg(all(feature = "enterprise", not(feature = "private")))] pub async fn verify_license_key() -> () { // Implementation is not open source } diff --git a/backend/src/main.rs b/backend/src/main.rs index a4148d93c6..be9a7578a3 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -28,7 +28,9 @@ use uuid::Uuid; use windmill_api::HTTP_CLIENT; #[cfg(feature = "enterprise")] -use windmill_common::ee::{maybe_renew_license_key_on_start, LICENSE_KEY_ID, LICENSE_KEY_VALID}; +use windmill_common::ee_oss::{ + maybe_renew_license_key_on_start, LICENSE_KEY_ID, LICENSE_KEY_VALID, +}; use windmill_common::{ agent_workers::build_agent_http_client, @@ -49,9 +51,12 @@ use windmill_common::{ TIMEOUT_WAIT_RESULT_SETTING, }, scripts::ScriptLang, - stats_ee::schedule_stats, + stats_oss::schedule_stats, triggers::TriggerKind, - utils::{hostname, rd_string, Mode, GIT_VERSION, MODE_AND_ADDONS}, + utils::{ + create_default_worker_suffix, create_ssh_agent_worker_suffix, worker_name_with_suffix, + Mode, GIT_VERSION, HOSTNAME, MODE_AND_ADDONS, + }, worker::{ reload_custom_tags_setting, Connection, HUB_CACHE_DIR, TMP_DIR, TMP_LOGS_DIR, WORKER_GROUP, }, @@ -75,8 +80,7 @@ use windmill_worker::{ get_hub_script_content_and_requirements, BUN_BUNDLE_CACHE_DIR, BUN_CACHE_DIR, CSHARP_CACHE_DIR, DENO_CACHE_DIR, DENO_CACHE_DIR_DEPS, DENO_CACHE_DIR_NPM, GO_BIN_CACHE_DIR, GO_CACHE_DIR, JAVA_CACHE_DIR, NU_CACHE_DIR, POWERSHELL_CACHE_DIR, PY310_CACHE_DIR, PY311_CACHE_DIR, - PY312_CACHE_DIR, PY313_CACHE_DIR, RUST_CACHE_DIR, TAR_JAVA_CACHE_DIR, TAR_PY310_CACHE_DIR, - TAR_PY311_CACHE_DIR, TAR_PY312_CACHE_DIR, TAR_PY313_CACHE_DIR, UV_CACHE_DIR, + PY312_CACHE_DIR, PY313_CACHE_DIR, RUST_CACHE_DIR, TAR_JAVA_CACHE_DIR, UV_CACHE_DIR, }; use crate::monitor::{ @@ -98,7 +102,9 @@ const DEFAULT_NUM_WORKERS: usize = 1; const DEFAULT_PORT: u16 = 8000; const DEFAULT_SERVER_BIND_ADDR: Ipv4Addr = Ipv4Addr::new(0, 0, 0, 0); -mod ee; +#[cfg(feature = "private")] +pub mod ee; +mod ee_oss; mod monitor; pub fn setup_deno_runtime() -> anyhow::Result<()> { @@ -153,6 +159,7 @@ lazy_static::lazy_static! { .ok() .and_then(|x| x.parse::().ok()) .unwrap_or(3600 * 12); + } pub fn main() -> anyhow::Result<()> { @@ -261,7 +268,7 @@ async fn windmill_main() -> anyhow::Result<()> { tracing::error!("Failed to install rustls crypto provider"); } - let hostname = hostname(); + let hostname = HOSTNAME.to_owned(); let mode_and_addons = MODE_AND_ADDONS.clone(); let mode = mode_and_addons.mode; @@ -340,7 +347,7 @@ async fn windmill_main() -> anyhow::Result<()> { "Creating http client for cluster using base internal url {}", std::env::var("BASE_INTERNAL_URL").unwrap_or_default() ); - let suffix = windmill_common::utils::worker_suffix(&hostname, &rd_string(5)); + let suffix = create_ssh_agent_worker_suffix(&hostname); ( Connection::Http(build_agent_http_client(&suffix)), Some(suffix), @@ -552,7 +559,7 @@ Windmill Community Edition {GIT_VERSION} _ = indexer_rx.recv() => { tracing::info!("Received killpill, aborting index initialization"); }, - res = windmill_indexer::completed_runs_ee::init_index(&db) => { + res = windmill_indexer::completed_runs_oss::init_index(&db) => { let res = res?; reader = Some(res.0); writer = Some(res.1); @@ -574,7 +581,7 @@ Windmill Community Edition {GIT_VERSION} async { if let Some(db) = conn.as_sql() { if let Some(index_writer) = index_writer2 { - windmill_indexer::completed_runs_ee::run_indexer( + windmill_indexer::completed_runs_oss::run_indexer( db.clone(), index_writer, indexer_rx, @@ -596,7 +603,7 @@ Windmill Community Edition {GIT_VERSION} _ = indexer_rx.recv() => { tracing::info!("Received killpill, aborting index initialization"); }, - res = windmill_indexer::service_logs_ee::init_index(&db, killpill_tx.clone()) => { + res = windmill_indexer::service_logs_oss::init_index(&db, killpill_tx.clone()) => { let res = res?; reader = Some(res.0); writer = Some(res.1); @@ -618,7 +625,7 @@ Windmill Community Edition {GIT_VERSION} async { if let Some(db) = conn.as_sql() { if let Some(log_index_writer) = log_index_writer2 { - windmill_indexer::service_logs_ee::run_indexer( + windmill_indexer::service_logs_oss::run_indexer( db.clone(), log_index_writer, log_indexer_rx, @@ -675,19 +682,21 @@ Windmill Community Edition {GIT_VERSION} let base_internal_url = base_internal_rx.await?; if worker_mode { let mut workers = vec![]; + for i in 0..num_workers { - let suffix: String = if i == 0 && first_suffix.as_ref().is_some() { + let suffix = if i == 0 && first_suffix.is_some() { first_suffix.as_ref().unwrap().clone() } else { - windmill_common::utils::worker_suffix(&hostname, &rd_string(5)) + create_default_worker_suffix(&hostname) }; + let worker_conn = WorkerConn { conn: if i == 0 || mode != Mode::Agent { conn.clone() } else { Connection::Http(build_agent_http_client(&suffix)) }, - worker_name: windmill_common::utils::worker_name_with_suffix( + worker_name: worker_name_with_suffix( mode == Mode::Agent, WORKER_GROUP.as_str(), &suffix, @@ -852,6 +861,11 @@ Windmill Community Edition {GIT_VERSION} } }; }, + "notify_token_invalidation" => { + let token = n.payload(); + tracing::info!("Token invalidation detected for token: {}...", &token[..token.len().min(8)]); + windmill_api::auth::invalidate_token_from_cache(token); + }, "notify_global_setting_change" => { tracing::info!("Global setting change detected: {}", n.payload()); match n.payload() { @@ -884,7 +898,7 @@ Windmill Community Edition {GIT_VERSION} if let Err(e) = load_tag_per_workspace_workspaces(&db).await { tracing::error!("Error loading default tag per workspace workspaces: {e:#}"); } - } + }, SMTP_SETTING => { reload_smtp_config(&db).await; }, @@ -1001,7 +1015,6 @@ Windmill Community Edition {GIT_VERSION} tracing::error!(error = %e, "Could not reload critical alert UI setting"); } }, - a @_ => { tracing::info!("Unrecognized Global Setting Change Payload: {:?}", a); } @@ -1053,7 +1066,9 @@ Windmill Community Edition {GIT_VERSION} } if server_mode { - tracing::info!("monitor task started"); + if !*windmill_common::QUIET_LOGS { + tracing::info!("monitor task started"); + } } monitor_db( &conn, @@ -1065,7 +1080,9 @@ Windmill Community Edition {GIT_VERSION} ) .await; if server_mode { - tracing::info!("monitor task finished"); + if !*windmill_common::QUIET_LOGS { + tracing::info!("monitor task finished"); + } } }, } @@ -1086,7 +1103,7 @@ Windmill Community Edition {GIT_VERSION} tracing::info!("Reloading config after 12 hours"); initial_load(&conn, tx.clone(), worker_mode, server_mode, #[cfg(feature = "parquet")] disable_s3_store).await; #[cfg(feature = "enterprise")] - ee::verify_license_key().await; + ee_oss::verify_license_key().await; } } }, @@ -1174,6 +1191,7 @@ async fn listen_pg(url: &str) -> Option { "notify_webhook_change", "notify_workspace_envs_change", "notify_runnable_version_change", + "notify_token_invalidation", ]; #[cfg(feature = "http_trigger")] @@ -1267,10 +1285,6 @@ pub async fn run_workers( PY311_CACHE_DIR, PY312_CACHE_DIR, PY313_CACHE_DIR, - TAR_PY310_CACHE_DIR, - TAR_PY311_CACHE_DIR, - TAR_PY312_CACHE_DIR, - TAR_PY313_CACHE_DIR, BUN_BUNDLE_CACHE_DIR, GO_CACHE_DIR, GO_BIN_CACHE_DIR, diff --git a/backend/src/monitor.rs b/backend/src/monitor.rs index d2a5657c6c..5c4677018c 100644 --- a/backend/src/monitor.rs +++ b/backend/src/monitor.rs @@ -29,9 +29,9 @@ use windmill_api::{ }; #[cfg(feature = "enterprise")] -use windmill_common::ee::low_disk_alerts; +use windmill_common::ee_oss::low_disk_alerts; #[cfg(feature = "enterprise")] -use windmill_common::ee::{jobs_waiting_alerts, worker_groups_alerts}; +use windmill_common::ee_oss::{jobs_waiting_alerts, worker_groups_alerts}; use windmill_common::client::AuthedClient; #[cfg(feature = "oauth2")] @@ -41,7 +41,7 @@ use windmill_common::s3_helpers::reload_object_store_setting; use windmill_common::{ agent_workers::DECODED_AGENT_TOKEN, auth::create_token_for_owner, - ee::CriticalErrorChannel, + ee_oss::CriticalErrorChannel, error, flow_status::{FlowStatus, FlowStatusModule}, global_settings::{ @@ -87,9 +87,9 @@ use windmill_worker::{ use windmill_common::s3_helpers::ObjectStoreReload; #[cfg(feature = "enterprise")] -use crate::ee::verify_license_key; +use crate::ee_oss::verify_license_key; -use crate::ee::set_license_key; +use crate::ee_oss::set_license_key; #[cfg(feature = "prometheus")] lazy_static::lazy_static! { @@ -1324,7 +1324,6 @@ pub async fn monitor_db( initial_load: bool, _killpill_tx: KillpillSender, ) { - tracing::info!("Starting periodic monitor task"); let zombie_jobs_f = async { if server_mode && !initial_load && !*DISABLE_ZOMBIE_JOBS_MONITORING { if let Some(db) = conn.as_sql() { @@ -1422,7 +1421,6 @@ pub async fn monitor_db( apply_autoscaling_f, update_min_worker_version_f, ); - tracing::info!("Periodic monitor task completed"); } pub async fn expose_queue_metrics(db: &Pool) { @@ -1609,7 +1607,7 @@ pub async fn reload_base_url_setting(conn: &Connection) -> error::Result<()> { if let Some(q) = q_oauth { if let Ok(v) = serde_json::from_value::< - Option>, + Option>, >(q.clone()) { v @@ -1630,7 +1628,7 @@ pub async fn reload_base_url_setting(conn: &Connection) -> error::Result<()> { { if let Some(db) = conn.as_sql() { let mut l = windmill_api::OAUTH_CLIENTS.write().await; - *l = windmill_api::oauth2_ee::build_oauth_clients(&base_url, oauths, db).await + *l = windmill_api::oauth2_oss::build_oauth_clients(&base_url, oauths, db).await .map_err(|e| tracing::error!("Error building oauth clients (is the oauth.json mounted and in correct format? Use '{}' as minimal oauth.json): {}", "{}", e)) .unwrap(); } @@ -1914,12 +1912,12 @@ async fn handle_zombie_jobs(db: &Pool, base_internal_url: &str, worker .await .expect("could not create job token"); - let client = AuthedClient { - base_internal_url: base_internal_url.to_string(), + let client = AuthedClient::new( + base_internal_url.to_string(), + job.workspace_id.to_string(), token, - workspace: job.workspace_id.to_string(), - force_client: None, - }; + None, + ); let last_ping = job.last_ping.clone(); let error_message = format!( @@ -1938,7 +1936,7 @@ async fn handle_zombie_jobs(db: &Pool, base_internal_url: &str, worker None, error::Error::ExecutionErr(error_message), true, - same_worker_tx_never_used, + Some(&same_worker_tx_never_used), "", worker_name, send_result_never_used, diff --git a/backend/substitute_ee_code.sh b/backend/substitute_ee_code.sh index 28b890990b..a1819c75b2 100755 --- a/backend/substitute_ee_code.sh +++ b/backend/substitute_ee_code.sh @@ -4,8 +4,8 @@ script_dirpath="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" root_dirpath="$(cd "${script_dirpath}/.." && pwd)" REVERT="NO" -REVERT_PREVIOUS="NO" COPY="NO" +MOVE_NEW_FILES="NO" EE_CODE_DIR="../windmill-ee-private/" while [[ $# -gt 0 ]]; do @@ -16,13 +16,7 @@ while [[ $# -gt 0 ]]; do # this to work (commit hooks should prevent this from happening, as well as the fact # that we're using symlinks by default). REVERT="YES" - shift - ;; - --revert-previous) - # This is a special case of --revert that will revert to the previous commit. - REVERT="YES" - REVERT_PREVIOUS="YES" - echo "Reverting to previous commit" + MOVE_NEW_FILES="YES" shift ;; -c|--copy) @@ -33,6 +27,11 @@ while [[ $# -gt 0 ]]; do COPY="YES" shift # past argument ;; + -m|--move-new-files) + # This moves all new EE files from the public repository to the private repository. + MOVE_NEW_FILES="YES" + shift # past argument + ;; -d|--dir) # Path to the local directory of the windmill-ee-private repository. By defaults, it # assumes it is cloned next to the Windmill OSS repo. @@ -70,29 +69,34 @@ if [ "$REVERT" == "YES" ]; then for ee_file in $(find ${EE_CODE_DIR} -name "*ee.rs"); do ce_file="${ee_file/${EE_CODE_DIR}/}" ce_file="${root_dirpath}/backend/${ce_file}" - if [ "$REVERT_PREVIOUS" == "YES" ]; then - git checkout HEAD@{3} ${ce_file} || true - else - git restore --staged ${ce_file} || true - git restore ${ce_file} || true - fi + rm ${ce_file} || true done -else +elif [ "$MOVE_NEW_FILES" == "NO" ]; then # This replaces all files in current repo with alternative EE files in windmill-ee-private for ee_file in $(find "${EE_CODE_DIR}" -name "*ee.rs"); do - ce_file="${ee_file/${EE_CODE_DIR}/}" - ce_file="${root_dirpath}/backend/${ce_file}" - if [[ -f "${ce_file}" ]]; then - rm "${ce_file}" - if [ "$COPY" == "YES" ]; then - cp "${ee_file}" "${ce_file}" - echo "File copied '${ee_file}' -->> '${ce_file}'" - else - ln -s "${ee_file}" "${ce_file}" - echo "Symlink created '${ee_file}' -->> '${ce_file}'" - fi + ce_file="${ee_file/${EE_CODE_DIR}/}" + ce_file="${root_dirpath}/backend/${ce_file}" + if [ "$COPY" == "YES" ]; then + cp "${ee_file}" "${ce_file}" + echo "File copied '${ee_file}' -->> '${ce_file}'" else - echo "File ${ce_file} is not a file, ignoring" + ln -s "${ee_file}" "${ce_file}" || true + echo "Symlink created '${ee_file}' -->> '${ce_file}'" fi done fi + +if [ "$MOVE_NEW_FILES" == "YES" ]; then + for ce_file in $(find "${root_dirpath}"/backend/windmill-*/src/ -name "*ee.rs"); do + backend_dirpath="${root_dirpath}/backend/" + ee_file="${ce_file/${backend_dirpath}/}" + ee_file="${EE_CODE_DIR}${ee_file}" + if [ ! -f "${ee_file}" ]; then + mv "${ce_file}" "${ee_file}" + if [ ! "$REVERT" == "YES" ]; then + ln -s "${ee_file}" "${ce_file}" + fi + echo "File moved '${ce_file}' -->> '${ee_file}'" + fi + done +fi \ No newline at end of file diff --git a/backend/summarize_schema.py b/backend/summarize_schema.py new file mode 100644 index 0000000000..a6d929ba9a --- /dev/null +++ b/backend/summarize_schema.py @@ -0,0 +1,154 @@ +# This script is used to summarize the database schema. +# You can use pg_dump to dump the schema to a file. +# pg_dump --file "schema.sql" --host "localhost" --port "5432" --username "postgres" --no-password --format=c --large-objects --schema-only --no-owner --no-privileges --no-tablespaces --no-unlogged-table-data --no-comments --no-publications --no-subscriptions --no-security-labels --no-toast-compression --no-table-access-method --verbose --schema "public" "windmill" +# Then you can run python summarize_schema.py schema.sql to get the summarized schema. + +import re +import sys +from collections import defaultdict + +def summarize_schema(file_path): + """ + Parses a PostgreSQL dump file and extracts a summarized schema. + """ + tables = defaultdict(lambda: {'columns': [], 'pks': set(), 'fks': [], 'indexes': []}) + enums = defaultdict(list) + + # Use state variables to parse multi-line definitions + current_table = None + current_enum = None + + with open(file_path, 'r', encoding='utf-8') as f: + for line in f: + line = line.strip() + + # --- State Resets --- + if line.startswith(');'): + current_table = None + current_enum = None + continue + + # --- Parse ENUM definitions --- + match_enum = re.match(r"CREATE TYPE public\.(\w+) AS ENUM \($", line) + if match_enum: + current_enum = match_enum.group(1) + continue + + if current_enum: + # Extract enum values, which are typically like 'value', + value = line.strip("',") + if value and not value.startswith('--'): + enums[current_enum].append(value) + continue + + # --- Parse TABLE definitions --- + match_table = re.match(r"CREATE TABLE public\.(\w+) \($", line) + if match_table: + current_table = match_table.group(1) + continue + + if current_table: + # Parse columns within a CREATE TABLE block + # e.g., "column_name type NOT NULL," + # e.g., "id bigint NOT NULL," + match_column = re.match(r'^"?(\w+)"?\s+([\w\d\.\[\]\(\)]+)', line) + if match_column: + col_name = match_column.group(1) + col_type = match_column.group(2) + tables[current_table]['columns'].append(f"{col_name} ({col_type})") + + # Parse PRIMARY KEY defined inside the table + match_pk = re.search(r"CONSTRAINT \w+ PRIMARY KEY \((.+)\)", line) + if match_pk: + # Handle multiple PK columns: "col1, col2, col3" + pk_cols = [p.strip().strip('"') for p in match_pk.group(1).split(',')] + tables[current_table]['pks'].update(pk_cols) + continue + + # --- Parse Foreign Keys (defined outside CREATE TABLE) --- + match_fk = re.match(r"ALTER TABLE ONLY public\.(\w+)\s+ADD CONSTRAINT \w+ FOREIGN KEY \(([\w,\s\"]+)\) REFERENCES public\.(\w+)\(([\w,\s\"]+)\);", line) + if match_fk: + from_table, from_cols, to_table, to_cols = match_fk.groups() + # Clean up column names + from_cols_clean = ', '.join([c.strip().strip('"') for c in from_cols.split(',')]) + to_cols_clean = ', '.join([c.strip().strip('"') for c in to_cols.split(',')]) + + fk_string = f"({from_cols_clean}) -> {to_table}({to_cols_clean})" + tables[from_table]['fks'].append(fk_string) + + # --- Parse Index definitions --- + match_index = re.match(r"CREATE (UNIQUE )?INDEX (\w+) ON public\.(\w+) USING (\w+) \((.+)\);", line) + if match_index: + is_unique = match_index.group(1) is not None + index_name = match_index.group(2) + table_name = match_index.group(3) + index_type = match_index.group(4) + columns = match_index.group(5) + + # Clean up column expressions + columns_clean = columns.replace('"', '') + + unique_str = "UNIQUE " if is_unique else "" + index_string = f"{unique_str}INDEX {index_name} ({index_type}) ON ({columns_clean})" + tables[table_name]['indexes'].append(index_string) + + return enums, tables + +def format_output(enums, tables): + """ + Formats the parsed schema data into a clean, readable string. + """ + output = [] + + output.append("### Simplified Database Schema ###") + output.append("\n--- Custom Data Types (ENUMs) ---\n") + if not enums: + output.append("No custom ENUM types found.") + else: + for name, values in sorted(enums.items()): + output.append(f"{name}:") + for v in values: + output.append(f" - {v}") + output.append("") + + output.append("\n--- Tables and Relationships ---\n") + if not tables: + output.append("No tables found.") + else: + for name, data in sorted(tables.items()): + output.append(f"TABLE: {name}") + for col in data['columns']: + col_name = col.split(' ')[0] + marker = " (PK)" if col_name in data['pks'] else "" + output.append(f" - {col}{marker}") + + if data['fks']: + output.append(" Relationships:") + for fk in data['fks']: + output.append(f" - {fk}") + + if data['indexes']: + output.append(" Indexes:") + for idx in data['indexes']: + output.append(f" - {idx}") + output.append("-" * 20) + + return "\n".join(output) + +if __name__ == "__main__": + if len(sys.argv) != 2: + print(f"Usage: python {sys.argv[0]} ") + sys.exit(1) + + input_file = sys.argv[1] + + try: + enums_data, tables_data = summarize_schema(input_file) + formatted_summary = format_output(enums_data, tables_data) + print(formatted_summary) + except FileNotFoundError: + print(f"Error: The file '{input_file}' was not found.") + sys.exit(1) + except Exception as e: + print(f"An unexpected error occurred: {e}") + sys.exit(1) \ No newline at end of file diff --git a/backend/summarized_schema.txt b/backend/summarized_schema.txt new file mode 100644 index 0000000000..a1ae0c1847 --- /dev/null +++ b/backend/summarized_schema.txt @@ -0,0 +1,1085 @@ +### Simplified Database Schema ### + +--- Custom Data Types (ENUMs) --- + +action_kind: + - create + - update + - delete + - execute + +authentication_method: + - none + - windmill + - api_key + - basic_http + - custom_script + - signature + +autoscaling_event_type: + - full_scaleout + - scalein + - scaleout + +aws_auth_resource_type: + - oidc + - credentials + +delivery_mode: + - push + - pull + +draft_type: + - script + - flow + - app + +favorite_kind: + - app + - script + - flow + - raw_app + +gcp_subscription_mode: + - create_update + - existing + +http_method: + - get + - post + - put + - delete + - patch + +importer_kind: + - script + - flow + - app + +job_kind: + - script + - preview + - flow + - dependencies + - flowpreview + - script_hub + - identity + - flowdependencies + - http + - graphql + - postgresql + - noop + - appdependencies + - deploymentcallback + - singlescriptflow + - flowscript + - flownode + - appscript + +job_status: + - success + - failure + - canceled + - skipped + +job_trigger_kind: + - webhook + - http + - websocket + - kafka + - email + - nats + - schedule + - app + - ui + - postgres + - sqs + - gcp + +log_mode: + - standalone + - server + - worker + - agent + - indexer + - mcp + +login_type: + - password + - github + +metric_kind: + - scalar_int + - scalar_float + - timeseries_int + - timeseries_float + +mqtt_client_version: + - v3 + - v5 + +runnable_type: + - ScriptHash + - ScriptPath + - FlowPath + +script_kind: + - script + - trigger + - failure + - command + - approval + - preprocessor + +script_lang: + - python3 + - deno + - go + - bash + - postgresql + - nativets + - bun + - mysql + - bigquery + - snowflake + - graphql + - powershell + - mssql + - php + - bunnative + - rust + - ansible + - csharp + - oracledb + - nu + - java + - duckdb + +trigger_kind: + - webhook + - http + - websocket + - kafka + - email + - nats + - postgres + - sqs + - mqtt + - gcp + +workspace_key_kind: + - cloud + + +--- Tables and Relationships --- + +TABLE: _sqlx_migrations + - version (bigint) + - description (text) + - installed_on (timestamp) + - success (boolean) + - checksum (bytea) + - execution_time (bigint) +-------------------- +TABLE: account + - workspace_id (character) + - id (integer) + - expires_at (timestamp) + - refresh_token (character) + - client (character) + - refresh_error (text) +-------------------- +TABLE: alerts + - id (integer) + - alert_type (character) + - message (text) + - created_at (timestamp) + - acknowledged (boolean) + - workspace_id (text) + - acknowledged_workspace (boolean) + - resource (text) + Indexes: + - INDEX alerts_by_workspace (btree) ON (workspace_id) +-------------------- +TABLE: app + - id (bigint) + - workspace_id (character) + - path (character) + - summary (character) + - policy (jsonb) + - versions (bigint[]) + - extra_perms (jsonb) + - draft_only (boolean) + - custom_path (text) + - CONSTRAINT (app_custom_path_check) +-------------------- +TABLE: app_script + - id (bigint) + - app (bigint) + - hash (character(64)) + - lock (text) + - code (text) + - code_sha256 (character(64)) +-------------------- +TABLE: app_version + - id (bigint) + - app_id (bigint) + - value (json) + - created_by (character) + - created_at (timestamp) + - raw_app (boolean) +-------------------- +TABLE: app_version_lite + - id (bigint) + - value (jsonb) +-------------------- +TABLE: audit + - workspace_id (character) + - id (integer) + - timestamp (timestamp) + - username (character) + - operation (character) + - action_kind (public.action_kind) + - resource (character) + - parameters (jsonb) + Indexes: + - INDEX ix_audit_timestamps (btree) ON (timestamp DESC) +-------------------- +TABLE: autoscaling_event + - id (integer) + - worker_group (text) + - event_type (public.autoscaling_event_type) + - desired_workers (integer) + - applied_at (timestamp) + - reason (text) + Indexes: + - INDEX autoscaling_event_worker_group_idx (btree) ON (worker_group, applied_at) +-------------------- +TABLE: capture + - workspace_id (character) + - path (character) + - created_at (timestamp) + - created_by (character) + - main_args (jsonb) + - is_flow (boolean) + - trigger_kind (public.trigger_kind) + - preprocessor_args (jsonb) + - id (bigint) + - CONSTRAINT (capture_payload_too_big) +-------------------- +TABLE: capture_config + - workspace_id (character) + - path (character) + - is_flow (boolean) + - trigger_kind (public.trigger_kind) + - trigger_config (jsonb) + - owner (character) + - email (character) + - server_id (character) + - last_client_ping (timestamp) + - last_server_ping (timestamp) + - error (text) +-------------------- +TABLE: cloud_workspace_settings + - workspace_id (character) + - threshold_alert_amount (integer) + - last_alert_sent (timestamp) + - last_warning_sent (timestamp) +-------------------- +TABLE: concurrency_counter + - concurrency_id (character) + - job_uuids (jsonb) +-------------------- +TABLE: concurrency_key + - key (character) + - ended_at (timestamp) + - job_id (uuid) + Indexes: + - INDEX concurrency_key_ended_at_idx (btree) ON (key, ended_at DESC) +-------------------- +TABLE: concurrency_locks + - id (character) + - last_locked_at (timestamp) + - owner (character) +-------------------- +TABLE: config + - name (character) + - config (jsonb) +-------------------- +TABLE: custom_concurrency_key_ended + - key (character) + - ended_at (timestamp) +-------------------- +TABLE: dependency_map + - workspace_id (character) + - importer_path (character) + - importer_kind (public.importer_kind) + - imported_path (character) + - importer_node_id (character) + Indexes: + - INDEX dependency_map_imported_path_idx (btree) ON (workspace_id, imported_path) +-------------------- +TABLE: deployment_metadata + - workspace_id (character) + - path (character) + - script_hash (bigint) + - app_version (bigint) + - callback_job_ids (uuid[]) + - deployment_msg (text) + - flow_version (bigint) + Indexes: + - UNIQUE INDEX deployment_metadata_app (btree) ON (workspace_id, path, app_version) WHERE (app_version IS NOT NULL) + - UNIQUE INDEX deployment_metadata_flow (btree) ON (workspace_id, path, flow_version) WHERE (flow_version IS NOT NULL) + - UNIQUE INDEX deployment_metadata_script (btree) ON (workspace_id, script_hash) WHERE (script_hash IS NOT NULL) +-------------------- +TABLE: draft + - workspace_id (character) + - path (character) + - typ (public.draft_type) + - value (json) + - created_at (timestamp) +-------------------- +TABLE: email_to_igroup + - email (character) + - igroup (character) +-------------------- +TABLE: favorite + - usr (character) + - workspace_id (character) + - path (character) + - favorite_kind (public.favorite_kind) +-------------------- +TABLE: flow + - workspace_id (character) + - path (character) + - summary (text) + - description (text) + - value (jsonb) + - edited_by (character) + - edited_at (timestamp) + - archived (boolean) + - schema (json) + - extra_perms (jsonb) + - dependency_job (uuid) + - draft_only (boolean) + - tag (character) + - ws_error_handler_muted (boolean) + - dedicated_worker (boolean) + - timeout (integer) + - visible_to_runner_only (boolean) + - concurrency_key (character) + - versions (bigint[]) + - on_behalf_of_email (text) + - lock_error_logs (text) + - CONSTRAINT (proper_id) + Indexes: + - INDEX flow_extra_perms (gin) ON (extra_perms) +-------------------- +TABLE: flow_node + - id (bigint) + - workspace_id (character) + - hash (bigint) + - path (character) + - lock (text) + - code (text) + - flow (jsonb) + - hash_v2 (character(64)) + Indexes: + - INDEX flow_node_hash (btree) ON (hash) +-------------------- +TABLE: flow_version + - id (bigint) + - workspace_id (character) + - path (character) + - value (jsonb) + - schema (json) + - created_by (character) + - created_at (timestamp) + Indexes: + - INDEX index_flow_version_path_created_at (btree) ON (path, created_at) +-------------------- +TABLE: flow_version_lite + - id (bigint) + - value (jsonb) +-------------------- +TABLE: folder + - name (character) + - workspace_id (character) + - display_name (character) + - owners (character) + - extra_perms (jsonb) + - summary (text) + - edited_at (timestamp) + - created_by (character) + Indexes: + - INDEX folder_extra_perms (gin) ON (extra_perms) + - INDEX folder_owners (gin) ON (owners) +-------------------- +TABLE: gcp_trigger + - gcp_resource_path (character) + - topic_id (character) + - subscription_id (character) + - delivery_type (public.delivery_mode) + - delivery_config (jsonb) + - path (character) + - script_path (character) + - is_flow (boolean) + - workspace_id (character) + - edited_by (character) + - email (character) + - edited_at (timestamp) + - extra_perms (jsonb) + - server_id (character) + - last_server_ping (timestamp) + - error (text) + - enabled (boolean) + - subscription_mode (public.gcp_subscription_mode) + - CONSTRAINT (gcp_trigger_check) + - CONSTRAINT (gcp_trigger_subscription_id_check) + - CONSTRAINT (gcp_trigger_topic_id_check) + Indexes: + - UNIQUE INDEX unique_subscription_per_gcp_resource (btree) ON (subscription_id, gcp_resource_path, workspace_id) +-------------------- +TABLE: global_settings + - name (character) + - value (jsonb) + - updated_at (timestamp) +-------------------- +TABLE: group_ + - workspace_id (character) + - name (character) + - summary (text) + - extra_perms (jsonb) + - CONSTRAINT (proper_name) + Indexes: + - INDEX group_extra_perms (gin) ON (extra_perms) +-------------------- +TABLE: healthchecks + - id (bigint) + - check_type (character) + - healthy (boolean) + - created_at (timestamp) + Indexes: + - INDEX healthchecks_check_type_created_at (btree) ON (check_type, created_at) +-------------------- +TABLE: http_trigger + - path (character) + - route_path (character) + - route_path_key (character) + - script_path (character) + - is_flow (boolean) + - workspace_id (character) + - edited_by (character) + - email (character) + - edited_at (timestamp) + - extra_perms (jsonb) + - is_async (boolean) + - authentication_method (public.authentication_method) + - http_method (public.http_method) + - static_asset_config (jsonb) + - is_static_website (boolean) + - workspaced_route (boolean) + - wrap_body (boolean) + - raw_string (boolean) + - authentication_resource_path (character) +-------------------- +TABLE: input + - id (uuid) + - workspace_id (character) + - runnable_id (character) + - runnable_type (public.runnable_type) + - name (text) + - args (jsonb) + - created_at (timestamp) + - created_by (character) + - is_public (boolean) +-------------------- +TABLE: instance_group + - name (character) + - summary (character) + - id (character) + - scim_display_name (character) + - external_id (character) +-------------------- +TABLE: job_logs + - job_id (uuid) + - workspace_id (character) + - created_at (timestamp) + - logs (text) + - log_offset (integer) + - log_file_index (text[]) +-------------------- +TABLE: job_perms + - job_id (uuid) + - email (character) + - username (character) + - is_admin (boolean) + - is_operator (boolean) + - created_at (timestamp) + - workspace_id (character) + - groups (text[]) + - folders (jsonb[]) +-------------------- +TABLE: job_stats + - workspace_id (character) + - job_id (uuid) + - metric_id (character) + - metric_name (character) + - metric_kind (public.metric_kind) + - scalar_int (integer) + - scalar_float (real) + - timestamps (timestamp) + - timeseries_int (integer[]) + - timeseries_float (real[]) + Indexes: + - INDEX job_stats_id (btree) ON (job_id) +-------------------- +TABLE: kafka_trigger + - path (character) + - kafka_resource_path (character) + - topics (character) + - group_id (character) + - script_path (character) + - is_flow (boolean) + - workspace_id (character) + - edited_by (character) + - email (character) + - edited_at (timestamp) + - extra_perms (jsonb) + - server_id (character) + - last_server_ping (timestamp) + - error (text) + - enabled (boolean) +-------------------- +TABLE: log_file + - hostname (character) + - log_ts (timestamp) + - ok_lines (bigint) + - err_lines (bigint) + - mode (public.log_mode) + - worker_group (character) + - file_path (character) + - json_fmt (boolean) + Indexes: + - INDEX log_file_log_ts_idx (btree) ON (log_ts) +-------------------- +TABLE: magic_link + - email (character) + - token (character) + - expiration (timestamp) + Indexes: + - INDEX index_magic_link_exp (btree) ON (expiration) +-------------------- +TABLE: metrics + - id (character) + - value (jsonb) + - created_at (timestamp) + Indexes: + - INDEX idx_metrics_id_created_at (btree) ON (id, created_at DESC) WHERE ((id)::text ~~ 'queue_%'::text) + - INDEX metrics_key_idx (btree) ON (id) + - INDEX metrics_sort_idx (btree) ON (created_at DESC) +-------------------- +TABLE: mqtt_trigger + - mqtt_resource_path (character) + - subscribe_topics (jsonb[]) + - client_version (public.mqtt_client_version) + - v5_config (jsonb) + - v3_config (jsonb) + - client_id (character) + - path (character) + - script_path (character) + - is_flow (boolean) + - workspace_id (character) + - edited_by (character) + - email (character) + - edited_at (timestamp) + - extra_perms (jsonb) + - server_id (character) + - last_server_ping (timestamp) + - error (text) + - enabled (boolean) +-------------------- +TABLE: nats_trigger + - path (character) + - nats_resource_path (character) + - subjects (character) + - stream_name (character) + - consumer_name (character) + - use_jetstream (boolean) + - script_path (character) + - is_flow (boolean) + - workspace_id (character) + - edited_by (character) + - email (character) + - edited_at (timestamp) + - extra_perms (jsonb) + - server_id (character) + - last_server_ping (timestamp) + - error (text) + - enabled (boolean) +-------------------- +TABLE: outstanding_wait_time + - job_id (uuid) + - self_wait_time_ms (bigint) + - aggregate_wait_time_ms (bigint) +-------------------- +TABLE: parallel_monitor_lock + - parent_flow_id (uuid) + - job_id (uuid) + - last_ping (timestamp) +-------------------- +TABLE: password + - email (character) + - password_hash (character) + - login_type (character) + - super_admin (boolean) + - verified (boolean) + - name (character) + - company (character) + - first_time_user (boolean) + - username (character) + - devops (boolean) +-------------------- +TABLE: pending_user + - email (character) + - created_at (timestamp) + - username (character) +-------------------- +TABLE: pip_resolution_cache + - hash (character) + - expiration (timestamp) + - lockfile (text) +-------------------- +TABLE: postgres_trigger + - path (character) + - script_path (character) + - is_flow (boolean) + - workspace_id (character) + - edited_by (character) + - email (character) + - edited_at (timestamp) + - extra_perms (jsonb) + - postgres_resource_path (character) + - error (text) + - server_id (character) + - last_server_ping (timestamp) + - replication_slot_name (character) + - publication_name (character) + - enabled (boolean) +-------------------- +TABLE: raw_app + - path (character) + - version (integer) + - workspace_id (character) + - summary (character) + - edited_at (timestamp) + - data (text) + - extra_perms (jsonb) +-------------------- +TABLE: resource + - workspace_id (character) + - path (character) + - value (jsonb) + - description (text) + - resource_type (character) + - extra_perms (jsonb) + - edited_at (timestamp) + - created_by (character) + - CONSTRAINT (proper_id) + Indexes: + - INDEX resource_extra_perms (gin) ON (extra_perms) +-------------------- +TABLE: resource_type + - workspace_id (character) + - name (character) + - schema (jsonb) + - description (text) + - edited_at (timestamp) + - created_by (character) + - format_extension (character) + - CONSTRAINT (proper_name) +-------------------- +TABLE: resume_job + - id (uuid) + - job (uuid) + - flow (uuid) + - created_at (timestamp) + - value (jsonb) + - approver (character) + - resume_id (integer) + - approved (boolean) +-------------------- +TABLE: schedule + - workspace_id (character) + - path (character) + - edited_by (character) + - edited_at (timestamp) + - schedule (character) + - enabled (boolean) + - script_path (character) + - args (jsonb) + - extra_perms (jsonb) + - is_flow (boolean) + - email (character) + - error (text) + - timezone (character) + - on_failure (character) + - on_recovery (character) + - on_failure_times (integer) + - on_failure_exact (boolean) + - on_failure_extra_args (jsonb) + - on_recovery_times (integer) + - on_recovery_extra_args (jsonb) + - ws_error_handler_muted (boolean) + - retry (jsonb) + - summary (character) + - no_flow_overlap (boolean) + - tag (character) + - paused_until (timestamp) + - on_success (character) + - on_success_extra_args (jsonb) + - cron_version (text) + - description (text) + - CONSTRAINT (proper_id) + Indexes: + - INDEX schedule_extra_perms (gin) ON (extra_perms) +-------------------- +TABLE: script + - workspace_id (character) + - hash (bigint) + - path (character) + - parent_hashes (bigint[]) + - summary (text) + - description (text) + - content (text) + - created_by (character) + - created_at (timestamp) + - archived (boolean) + - schema (json) + - deleted (boolean) + - is_template (boolean) + - extra_perms (jsonb) + - lock (text) + - lock_error_logs (text) + - language (public.script_lang) + - kind (public.script_kind) + - tag (character) + - draft_only (boolean) + - envs (character) + - concurrent_limit (integer) + - concurrency_time_window_s (integer) + - cache_ttl (integer) + - dedicated_worker (boolean) + - ws_error_handler_muted (boolean) + - priority (smallint) + - timeout (integer) + - delete_after_use (boolean) + - restart_unless_cancelled (boolean) + - concurrency_key (character) + - visible_to_runner_only (boolean) + - no_main_func (boolean) + - codebase (character) + - has_preprocessor (boolean) + - on_behalf_of_email (text) + - schema_validation (boolean) + - CONSTRAINT (proper_id) + Indexes: + - INDEX index_script_on_path_created_at (btree) ON (workspace_id, path, created_at DESC) + - INDEX script_extra_perms (gin) ON (extra_perms) +-------------------- +TABLE: sqs_trigger + - path (character) + - queue_url (character) + - aws_resource_path (character) + - message_attributes (text[]) + - script_path (character) + - is_flow (boolean) + - workspace_id (character) + - edited_by (character) + - email (character) + - edited_at (timestamp) + - extra_perms (jsonb) + - error (text) + - server_id (character) + - last_server_ping (timestamp) + - enabled (boolean) + - aws_auth_resource_type (public.aws_auth_resource_type) +-------------------- +TABLE: token + - token (character) + - label (character) + - expiration (timestamp) + - workspace_id (character) + - owner (character) + - email (character) + - super_admin (boolean) + - created_at (timestamp) + - last_used_at (timestamp) + - scopes (text[]) + - job (uuid) + Indexes: + - INDEX index_token_exp (btree) ON (expiration) +-------------------- +TABLE: tutorial_progress + - email (character) + - progress (bit(64)) +-------------------- +TABLE: usage + - id (character) + - is_workspace (boolean) + - month_ (integer) + - usage (integer) +-------------------- +TABLE: usr + - workspace_id (character) + - username (character) + - email (character) + - is_admin (boolean) + - created_at (timestamp) + - operator (boolean) + - disabled (boolean) + - role (character) + - CONSTRAINT (proper_email) + - CONSTRAINT (proper_username) + Indexes: + - INDEX index_usr_email (btree) ON (email) +-------------------- +TABLE: usr_to_group + - workspace_id (character) + - group_ (character) + - usr (character) +-------------------- +TABLE: v2_job + - id (uuid) + - raw_code (text) + - raw_lock (text) + - raw_flow (jsonb) + - tag (character) + - workspace_id (character) + - created_at (timestamp) + - created_by (character) + - permissioned_as (character) + - permissioned_as_email (character) + - kind (public.job_kind) + - runnable_id (bigint) + - runnable_path (character) + - parent_job (uuid) + - root_job (uuid) + - script_lang (public.script_lang) + - script_entrypoint_override (character) + - flow_step (integer) + - flow_step_id (character) + - flow_innermost_root_job (uuid) + - trigger (character) + - trigger_kind (public.job_trigger_kind) + - same_worker (boolean) + - visible_to_owner (boolean) + - concurrent_limit (integer) + - concurrency_time_window_s (integer) + - cache_ttl (integer) + - timeout (integer) + - priority (smallint) + - preprocessed (boolean) + - args (jsonb) + - labels (text[]) + - pre_run_error (text) + Indexes: + - INDEX ix_job_created_at (btree) ON (created_at DESC) + - INDEX ix_job_root_job_index_by_path_2 (btree) ON (workspace_id, runnable_path, created_at DESC) WHERE (parent_job IS NULL) + - INDEX ix_job_workspace_id_created_at_new_3 (btree) ON (workspace_id, created_at DESC) + - INDEX ix_job_workspace_id_created_at_new_5 (btree) ON (workspace_id, created_at DESC) WHERE ((kind = ANY (ARRAY['preview'::public.job_kind, 'flowpreview'::public.job_kind])) AND (parent_job IS NULL)) + - INDEX ix_job_workspace_id_created_at_new_8 (btree) ON (workspace_id, created_at DESC) WHERE ((kind = 'deploymentcallback'::public.job_kind) AND (parent_job IS NULL)) + - INDEX ix_job_workspace_id_created_at_new_9 (btree) ON (workspace_id, created_at DESC) WHERE ((kind = ANY (ARRAY['dependencies'::public.job_kind, 'flowdependencies'::public.job_kind, 'appdependencies'::public.job_kind])) AND (parent_job IS NULL)) + - INDEX ix_v2_job_labels (gin) ON (labels) WHERE (labels IS NOT NULL) + - INDEX ix_v2_job_workspace_id_created_at (btree) ON (workspace_id, created_at DESC) WHERE ((kind = ANY (ARRAY['script'::public.job_kind, 'flow'::public.job_kind, 'singlescriptflow'::public.job_kind])) AND (parent_job IS NULL)) +-------------------- +TABLE: v2_job_completed + - id (uuid) + - workspace_id (character) + - duration_ms (bigint) + - result (jsonb) + - deleted (boolean) + - canceled_by (character) + - canceled_reason (text) + - flow_status (jsonb) + - started_at (timestamp) + - memory_peak (integer) + - status (public.job_status) + - completed_at (timestamp) + - worker (character) + - workflow_as_code_status (jsonb) + - result_columns (text[]) + - retries (uuid[]) + - extras (jsonb) + Indexes: + - INDEX ix_completed_job_workspace_id_started_at_new_2 (btree) ON (workspace_id, started_at DESC) + - INDEX ix_job_completed_completed_at (btree) ON (completed_at DESC) + - INDEX labeled_jobs_on_jobs (gin) ON (((result -> 'wm_labels'::text))) WHERE (result ? 'wm_labels'::text) +-------------------- +TABLE: v2_job_queue + - id (uuid) + - workspace_id (character) + - created_at (timestamp) + - started_at (timestamp) + - scheduled_for (timestamp) + - running (boolean) + - canceled_by (character) + - canceled_reason (text) + - suspend (integer) + - suspend_until (timestamp) + - tag (character) + - priority (smallint) + - worker (character) + - extras (jsonb) + Indexes: + - INDEX queue_sort_v2 (btree) ON (priority DESC NULLS LAST, scheduled_for, tag) WHERE (running = false) + - INDEX queue_suspended (btree) ON (priority DESC NULLS LAST, created_at, suspend_until, suspend, tag) WHERE (suspend_until IS NOT NULL) + - INDEX root_queue_index_by_path (btree) ON (workspace_id, created_at) + - INDEX v2_job_queue_suspend (btree) ON (workspace_id, suspend) WHERE (suspend > 0) +-------------------- +TABLE: v2_job_runtime + - id (uuid) + - ping (timestamp) + - memory_peak (integer) +-------------------- +TABLE: v2_job_status + - id (uuid) + - flow_status (jsonb) + - flow_leaf_jobs (jsonb) + - workflow_as_code_status (jsonb) +-------------------- +TABLE: variable + - workspace_id (character) + - path (character) + - value (character) + - is_secret (boolean) + - description (character) + - extra_perms (jsonb) + - account (integer) + - is_oauth (boolean) + - expires_at (timestamp) + - CONSTRAINT (proper_id) + Indexes: + - INDEX variable_extra_perms (gin) ON (extra_perms) +-------------------- +TABLE: websocket_trigger + - path (character) + - url (character) + - script_path (character) + - is_flow (boolean) + - workspace_id (character) + - edited_by (character) + - email (character) + - edited_at (timestamp) + - extra_perms (jsonb) + - server_id (character) + - last_server_ping (timestamp) + - error (text) + - enabled (boolean) + - filters (jsonb[]) + - initial_messages (jsonb[]) + - url_runnable_args (jsonb) + - can_return_message (boolean) +-------------------- +TABLE: windmill_migrations + - name (text) + - created_at (timestamp) +-------------------- +TABLE: worker_ping + - worker (character) + - worker_instance (character) + - ping_at (timestamp) + - started_at (timestamp) + - ip (character) + - jobs_executed (integer) + - custom_tags (text[]) + - worker_group (character) + - dedicated_worker (character) + - wm_version (character) + - current_job_id (uuid) + - current_job_workspace_id (character) + - vcpus (bigint) + - memory (bigint) + - occupancy_rate (real) + - memory_usage (bigint) + - wm_memory_usage (bigint) + - occupancy_rate_15s (real) + - occupancy_rate_5m (real) + - occupancy_rate_30m (real) + Indexes: + - INDEX worker_ping_on_ping_at (btree) ON (ping_at) +-------------------- +TABLE: workspace + - id (character) + - name (character) + - owner (character) + - deleted (boolean) + - premium (boolean) + - CONSTRAINT (proper_id) +-------------------- +TABLE: workspace_env + - workspace_id (character) + - name (character) + - value (character) +-------------------- +TABLE: workspace_invite + - workspace_id (character) + - email (character) + - is_admin (boolean) + - operator (boolean) + - CONSTRAINT (proper_email) +-------------------- +TABLE: workspace_key + - workspace_id (character) + - kind (public.workspace_key_kind) + - key (character) +-------------------- +TABLE: workspace_runnable_dependencies + - flow_path (character) + - runnable_path (character) + - script_hash (bigint) + - runnable_is_flow (boolean) + - workspace_id (character) + - app_path (character) + - CONSTRAINT (workspace_runnable_dependencies_path_exclusive) + Indexes: + - UNIQUE INDEX app_workspace_with_hash_unique_idx (btree) ON (app_path, runnable_path, script_hash, runnable_is_flow, workspace_id) WHERE (script_hash IS NOT NULL) + - UNIQUE INDEX app_workspace_without_hash_unique_idx (btree) ON (app_path, runnable_path, runnable_is_flow, workspace_id) WHERE (script_hash IS NULL) + - INDEX flow_workspace_runnable_path_is_flow_idx (btree) ON (runnable_path, runnable_is_flow, workspace_id) + - UNIQUE INDEX flow_workspace_with_hash_unique_idx (btree) ON (flow_path, runnable_path, script_hash, runnable_is_flow, workspace_id) WHERE (script_hash IS NOT NULL) + - UNIQUE INDEX flow_workspace_without_hash_unique_idx (btree) ON (flow_path, runnable_path, runnable_is_flow, workspace_id) WHERE (script_hash IS NULL) +-------------------- +TABLE: workspace_settings + - workspace_id (character) + - slack_team_id (character) + - slack_name (character) + - slack_command_script (character) + - slack_email (character) + - auto_invite_domain (character) + - auto_invite_operator (boolean) + - customer_id (character) + - plan (character) + - webhook (text) + - deploy_to (character) + - error_handler (character) + - ai_config (jsonb) + - error_handler_extra_args (json) + - error_handler_muted_on_cancel (boolean) + - large_file_storage (jsonb) + - git_sync (jsonb) + - default_app (character) + - auto_add (boolean) + - default_scripts (jsonb) + - deploy_ui (jsonb) + - mute_critical_alerts (boolean) + - color (character) + - operator_settings (jsonb) + - teams_command_script (text) + - teams_team_id (text) + - teams_team_name (text) + - git_app_installations (jsonb) +-------------------- +TABLE: zombie_job_counter + - job_id (uuid) + - counter (integer) +-------------------- diff --git a/backend/tests/worker.rs b/backend/tests/worker.rs index c8299796f8..8792b2eb1d 100644 --- a/backend/tests/worker.rs +++ b/backend/tests/worker.rs @@ -3969,8 +3969,7 @@ async fn assert_lockfile( #[cfg(feature = "python")] #[sqlx::test(fixtures("base", "lockfile_python"))] async fn test_requirements_python(db: Pool) { - let content = r#" -# py: 3.11.11 + let content = r#"# py: ==3.11.11 # requirements: # tiny==0.1.3 @@ -3997,8 +3996,7 @@ def main(): #[sqlx::test(fixtures("base", "lockfile_python"))] async fn test_extra_requirements_python(db: Pool) { { - let content = r#" -# py: ==3.11.11 + let content = r#"# py: ==3.11.11 # extra_requirements: # tiny @@ -4025,8 +4023,7 @@ def main(): #[cfg(feature = "python")] #[sqlx::test(fixtures("base", "lockfile_python"))] async fn test_extra_requirements_python2(db: Pool) { - let content = r#" -# py: ==3.11.11 + let content = r#"# py: ==3.11.11 # extra_requirements: # tiny==0.1.3 @@ -4048,8 +4045,7 @@ def main(): #[cfg(feature = "python")] #[sqlx::test(fixtures("base", "lockfile_python"))] async fn test_pins_python(db: Pool) { - let content = r#" -# py: ==3.11.11 + let content = r#"# py: ==3.11.11 # extra_requirements: # tiny==0.1.3 # bottle==0.13.2 diff --git a/backend/update_sqlx.sh b/backend/update_sqlx.sh index 11b5196ad8..09824d9e24 100755 --- a/backend/update_sqlx.sh +++ b/backend/update_sqlx.sh @@ -10,7 +10,6 @@ if [[ "$(uname)" == "Darwin" ]]; then fi cargo sqlx prepare --workspace -- --all-targets --all-features -./substitute_ee_code.sh -r --dir ../windmill-ee-private # Undo the samael changes on macOS if [[ "$(uname)" == "Darwin" ]]; then diff --git a/backend/windmill-api/Cargo.toml b/backend/windmill-api/Cargo.toml index 9f6de0c5ca..7a715d7437 100644 --- a/backend/windmill-api/Cargo.toml +++ b/backend/windmill-api/Cargo.toml @@ -10,13 +10,14 @@ path = "src/lib.rs" [features] default = [] +private = ["windmill-audit/private"] enterprise = ["windmill-queue/enterprise", "windmill-audit/enterprise", "windmill-git-sync/enterprise", "windmill-common/enterprise", "windmill-worker/enterprise"] stripe = [] agent_worker_server = [] enterprise_saml = ["dep:samael", "dep:libxml"] benchmark = [] embedding = ["dep:tinyvector", "dep:hf-hub", "dep:tokenizers", "dep:candle-core", "dep:candle-transformers", "dep:candle-nn"] -parquet = ["dep:datafusion", "dep:object_store", "dep:url", "windmill-common/parquet", "windmill-worker/parquet"] +parquet = ["dep:datafusion", "dep:object_store", "windmill-common/parquet", "windmill-worker/parquet"] prometheus = ["windmill-common/prometheus", "windmill-queue/prometheus", "dep:prometheus", "windmill-worker/prometheus"] openidconnect = ["dep:openidconnect", "windmill-common/openidconnect"] tantivy = ["dep:windmill-indexer"] @@ -39,7 +40,7 @@ mcp = ["dep:rmcp"] python = [] [dependencies] -rmcp = { git = "https://github.com/windmill-labs/rust-sdk", features = ["transport-sse-server"], optional = true } +rmcp = { git = "https://github.com/modelcontextprotocol/rust-sdk", features=["transport-streamable-http-server", "transport-streamable-http-server-session", "transport-worker"], optional = true } windmill-queue.workspace = true windmill-common = { workspace = true, default-features = false } windmill-audit.workspace = true @@ -75,6 +76,7 @@ hex.workspace = true base64.workspace = true base32.workspace = true serde_urlencoded.workspace = true +serde_yml.workspace = true cron.workspace = true mime_guess.workspace = true rust-embed = { workspace = true, optional = true } @@ -101,6 +103,7 @@ prometheus = { workspace = true, optional = true } async_zip = { workspace = true, optional = true } regex.workspace = true bytes.workspace = true +url.workspace = true samael = { workspace = true, optional = true } libxml = { workspace = true, optional = true } async-recursion.workspace = true @@ -115,7 +118,6 @@ candle-nn = { workspace = true, optional = true} datafusion = { workspace = true, optional = true} object_store = { workspace = true, optional = true} openidconnect = { workspace = true, optional = true} -url = { workspace = true, optional = true} jsonwebtoken = { workspace = true } matchit = { workspace = true, optional = true } tokio-tungstenite = { workspace = true, optional = true} @@ -125,6 +127,7 @@ nkeys = { workspace = true, optional = true } const_format.workspace = true pin-project.workspace = true http.workspace = true +indexmap.workspace = true async-stream.workspace = true ulid.workspace = true rust-postgres = { workspace = true, optional = true } diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index e757d24432..a5e693277e 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -1,7 +1,7 @@ openapi: "3.0.3" info: - version: 1.494.0 + version: 1.498.0 title: Windmill API contact: @@ -4501,6 +4501,13 @@ paths: in: query schema: type: boolean + - name: languages + in: query + description: | + Filter to only include scripts written in the given languages. + Accepts multiple values as a comma-separated list. + schema: + type: string responses: "200": description: All scripts @@ -8444,6 +8451,76 @@ paths: "201": description: default error handler set + /w/{workspace}/openapi/generate: + post: + summary: generate openapi spec from http routes/webhook + operationId: generateOpenapiSpec + tags: + - openapi + parameters: + - $ref: "#/components/parameters/WorkspaceId" + requestBody: + description: openapi spec info and url + content: + application/json: + schema: + $ref: "#/components/schemas/GenerateOpenapiSpec" + responses: + "200": + description: openapi spec + content: + text/plain: + schema: + type: string + + /w/{workspace}/openapi/download: + post: + summary: Download the OpenAPI v3.1 spec as a file + operationId: DownloadOpenapiSpec + tags: + - openapi + parameters: + - $ref: "#/components/parameters/WorkspaceId" + requestBody: + description: openapi spec info and url + content: + application/json: + schema: + $ref: "#/components/schemas/GenerateOpenapiSpec" + responses: + "200": + description: Downloaded OpenAPI spec + content: + application/octet-stream: + schema: + type: string + format: binary + + /w/{workspace}/http_triggers/create_many: + post: + summary: create many HTTP triggers + operationId: createHttpTriggers + tags: + - http_trigger + parameters: + - $ref: "#/components/parameters/WorkspaceId" + requestBody: + description: new http trigger + required: true + content: + application/json: + schema: + type: array + items: + $ref: "#/components/schemas/NewHttpTrigger" + responses: + "201": + description: http trigger created + content: + text/plain: + schema: + type: string + /w/{workspace}/http_triggers/create: post: summary: create http trigger @@ -8595,8 +8672,7 @@ paths: route_path: type: string http_method: - type: string - enum: ["get", "post", "put", "delete", "patch"] + $ref: "#/components/schemas/HttpMethod" trigger_path: type: string workspaced_route: @@ -13185,6 +13261,8 @@ components: # NOTE: Not so many generators and validators support this format: # $ref: "../../openflow.openapi.yaml#/components/schemas" # This is why it is better to inline each of schemas for better compat + # Do not change next line. It is used by python-client for pre-processing + # -- INLINE START -- OpenFlow: $ref: "../../openflow.openapi.yaml#/components/schemas/OpenFlow" FlowValue: @@ -13223,6 +13301,8 @@ components: $ref: "../../openflow.openapi.yaml#/components/schemas/FlowStatus" FlowStatusModule: $ref: "../../openflow.openapi.yaml#/components/schemas/FlowStatusModule" + # -- INLINE END -- + # Do not change line above AIProvider: type: string @@ -14719,6 +14799,115 @@ components: - custom_script - signature + RunnableKind: + type: string + enum: + - script + - flow + + OpenapiSpecFormat: + type: string + enum: + - yaml + - json + + OpenapiHttpRouteFilters: + type: object + properties: + folder_regex: + type: string + path_regex: + type: string + route_path_regex: + type: string + required: + - folder_regex + - path_regex + - route_path_regex + + WebhookFilters: + type: object + properties: + user_or_folder_regex: + type: string + enum: + - "*" + - u + - f + user_or_folder_regex_value: + type: string + path: + type: string + runnable_kind: + $ref: "#/components/schemas/RunnableKind" + required: + - user_or_folder_regex + - user_or_folder_regex_value + - path + - runnable_kind + + OpenapiV3Info: + type: object + properties: + title: + type: string + version: + type: string + description: + type: string + terms_of_service: + type: string + contact: + type: object + properties: + name: + type: string + url: + type: string + email: + type: string + license: + type: object + properties: + name: + type: string + identifier: + type: string + url: + type: string + required: + - name + required: + - title + - version + + GenerateOpenapiSpec: + type: object + properties: + info: + $ref: "#/components/schemas/OpenapiV3Info" + url: + type: string + openapi_spec_format: + $ref: "#/components/schemas/OpenapiSpecFormat" + http_route_filters: + type: array + items: + $ref: "#/components/schemas/OpenapiHttpRouteFilters" + webhook_filters: + type: array + items: + $ref: "#/components/schemas/WebhookFilters" + + HttpMethod: + type: string + enum: + - get + - post + - put + - delete + - patch + HttpTrigger: allOf: - $ref: "#/components/schemas/TriggerExtraProperty" @@ -14738,15 +14927,13 @@ components: required: - s3 http_method: - type: string - enum: - - get - - post - - put - - delete - - patch + $ref: "#/components/schemas/HttpMethod" authentication_resource_path: type: string + summary: + type: string + description: + type: string is_async: type: boolean authentication_method: @@ -14781,6 +14968,10 @@ components: type: string workspaced_route: type: boolean + summary: + type: string + description: + type: string static_asset_config: type: object properties: @@ -14795,13 +14986,7 @@ components: is_flow: type: boolean http_method: - type: string - enum: - - get - - post - - put - - delete - - patch + $ref: "#/components/schemas/HttpMethod" authentication_resource_path: type: string is_async: @@ -14834,6 +15019,10 @@ components: type: string route_path: type: string + summary: + type: string + description: + type: string workspaced_route: type: boolean static_asset_config: @@ -14852,13 +15041,7 @@ components: is_flow: type: boolean http_method: - type: string - enum: - - get - - post - - put - - delete - - patch + $ref: "#/components/schemas/HttpMethod" is_async: type: boolean authentication_method: diff --git a/backend/windmill-api/src/agent_workers_ee.rs b/backend/windmill-api/src/agent_workers_oss.rs similarity index 76% rename from backend/windmill-api/src/agent_workers_ee.rs rename to backend/windmill-api/src/agent_workers_oss.rs index 1b5a210d13..655058c493 100644 --- a/backend/windmill-api/src/agent_workers_ee.rs +++ b/backend/windmill-api/src/agent_workers_oss.rs @@ -1,3 +1,7 @@ +#[cfg(feature = "private")] +#[allow(unused)] +pub use crate::agent_workers_ee::*; + /* * Author: Ruben Fiszel * Copyright: Windmill Labs, Inc 2042 @@ -6,16 +10,21 @@ * LICENSE-AGPL for a copy of the license. */ +#[cfg(not(feature = "private"))] use crate::db::DB; +#[cfg(not(feature = "private"))] use axum::Router; +#[cfg(not(feature = "private"))] use serde::{Deserialize, Serialize}; +#[cfg(not(feature = "private"))] pub fn global_service() -> Router { Router::new() } +#[cfg(not(feature = "private"))] pub fn workspaced_service( db: DB, _base_internal_url: String, @@ -36,6 +45,7 @@ pub fn workspaced_service( } #[derive(Clone, Debug, Deserialize, Serialize)] +#[cfg(not(feature = "private"))] pub struct AgentAuth { pub worker_group: String, pub suffix: Option, @@ -43,8 +53,10 @@ pub struct AgentAuth { pub exp: Option, } +#[cfg(not(feature = "private"))] pub struct AgentCache {} +#[cfg(not(feature = "private"))] impl AgentCache { pub fn new() -> Self { AgentCache {} diff --git a/backend/windmill-api/src/ai.rs b/backend/windmill-api/src/ai.rs index 6b33a70730..44a6b2986e 100644 --- a/backend/windmill-api/src/ai.rs +++ b/backend/windmill-api/src/ai.rs @@ -4,13 +4,13 @@ use crate::{ }; use axum::{body::Bytes, extract::Path, response::IntoResponse, routing::post, Extension, Router}; -use http::HeaderMap; +use http::{HeaderMap, Method}; use quick_cache::sync::Cache; use reqwest::{Client, RequestBuilder}; use serde::{Deserialize, Serialize}; use serde_json::value::RawValue; use std::collections::HashMap; -use windmill_audit::{audit_ee::audit_log, ActionKind}; +use windmill_audit::{audit_oss::audit_log, ActionKind}; use windmill_common::error::{to_anyhow, Error, Result}; lazy_static::lazy_static! { @@ -24,7 +24,7 @@ lazy_static::lazy_static! { pub static ref AI_REQUEST_CACHE: Cache<(String, AIProvider), ExpiringAIRequestConfig> = Cache::new(500); } -const AZURE_API_VERSION: &str = "2024-10-21"; +const AZURE_API_VERSION: &str = "2025-04-01-preview"; const OPENAI_BASE_URL: &str = "https://api.openai.com/v1"; #[derive(Deserialize, Debug)] @@ -141,23 +141,49 @@ impl AIRequestConfig { self, provider: &AIProvider, path: &str, + method: Method, + headers: HeaderMap, body: Bytes, ) -> Result { - let url = format!("{}/{}", self.base_url, path); - let body = if let Some(user) = self.user { Self::add_user_to_body(body, user)? } else { body }; - let is_azure = matches!(provider, AIProvider::OpenAI) && self.base_url != OPENAI_BASE_URL + let base_url = self.base_url.trim_end_matches('/'); + + let is_azure = matches!(provider, AIProvider::OpenAI) && base_url != OPENAI_BASE_URL || matches!(provider, AIProvider::AzureOpenAI); + let is_anthropic = matches!(provider, AIProvider::Anthropic); + + let url = if is_azure && method != Method::GET { + if base_url.ends_with("/deployments") { + let model = Self::get_azure_model(&body)?; + format!("{}/{}/{}", base_url, model, path) + } else if base_url.ends_with("/openai") { + let model = Self::get_azure_model(&body)?; + format!("{}/deployments/{}/{}", base_url, model, path) + } else { + format!("{}/{}", base_url, path) + } + } else { + format!("{}/{}", base_url, path) + }; + + tracing::debug!("AI request URL: {}", url); let mut request = HTTP_CLIENT - .post(url) - .header("content-type", "application/json") - .body(body); + .request(method, url) + .header("content-type", "application/json"); + + for (header_name, header_value) in headers.iter() { + if header_name.to_string().starts_with("anthropic-") { + request = request.header(header_name, header_value); + } + } + + request = request.body(body); if is_azure { request = request.query(&[("api-version", AZURE_API_VERSION)]) @@ -165,9 +191,12 @@ impl AIRequestConfig { if let Some(api_key) = self.api_key { if is_azure { - request = request.header("api-key", api_key) + request = request.header("api-key", api_key.clone()) } else { - request = request.header("authorization", format!("Bearer {}", api_key)) + request = request.header("authorization", format!("Bearer {}", api_key.clone())) + } + if is_anthropic { + request = request.header("X-API-Key", api_key); } } @@ -199,6 +228,18 @@ impl AIRequestConfig { .map_err(|e| Error::internal_err(format!("Failed to reserialize request body: {}", e)))? .into()) } + + fn get_azure_model(body: &Bytes) -> Result { + #[derive(Deserialize, Debug)] + struct AzureModel { + model: String, + } + + let azure_model: AzureModel = serde_json::from_slice(body) + .map_err(|e| Error::internal_err(format!("Failed to parse request body: {}", e)))?; + + Ok(azure_model.model) + } } #[derive(Clone, Debug)] @@ -311,17 +352,18 @@ pub struct AIConfig { } pub fn global_service() -> Router { - Router::new().route("/proxy/*ai", post(global_proxy)) + Router::new().route("/proxy/*ai", post(global_proxy).get(global_proxy)) } pub fn workspaced_service() -> Router { - Router::new().route("/proxy/*ai", post(proxy)) + Router::new().route("/proxy/*ai", post(proxy).get(proxy)) } async fn global_proxy( authed: ApiAuthed, Extension(db): Extension, Path(ai_path): Path, + method: Method, headers: HeaderMap, body: Bytes, ) -> impl IntoResponse { @@ -346,7 +388,7 @@ async fn global_proxy( let url = format!("{}/{}", base_url, ai_path); let request = HTTP_CLIENT - .post(url) + .request(method, url) .header("content-type", "application/json") .header("Authorization", format!("Bearer {}", api_key)) .body(body); @@ -382,6 +424,7 @@ async fn proxy( authed: ApiAuthed, Extension(db): Extension, Path((w_id, ai_path)): Path<(String, String)>, + method: Method, headers: HeaderMap, body: Bytes, ) -> impl IntoResponse { @@ -464,7 +507,7 @@ async fn proxy( } }; - let request = request_config.prepare_request(&provider, &ai_path, body)?; + let request = request_config.prepare_request(&provider, &ai_path, method, headers, body)?; let response = request.send().await.map_err(to_anyhow)?; diff --git a/backend/windmill-api/src/apps.rs b/backend/windmill-api/src/apps.rs index 533da11d1c..befe569229 100644 --- a/backend/windmill-api/src/apps.rs +++ b/backend/windmill-api/src/apps.rs @@ -12,13 +12,13 @@ use crate::{ db::{ApiAuthed, DB}, resources::get_resource_value_interpolated_internal, users::{require_owner_of_path, OptAuthed}, - utils::{RunnableKind, WithStarredInfoQuery}, + utils::WithStarredInfoQuery, webhook_util::{WebhookMessage, WebhookShared}, HTTP_CLIENT, }; #[cfg(feature = "parquet")] use crate::{ - job_helpers_ee::{ + job_helpers_oss::{ download_s3_file_internal, get_random_file_name, get_s3_resource, get_workspace_s3_resource, upload_file_from_req, DownloadFileQuery, }, @@ -48,7 +48,7 @@ use sha2::{Digest, Sha256}; use sql_builder::{bind::Bind, SqlBuilder}; use sqlx::{types::Uuid, FromRow}; use std::str; -use windmill_audit::audit_ee::audit_log; +use windmill_audit::audit_oss::audit_log; use windmill_audit::ActionKind; use windmill_common::{ apps::{AppScriptId, ListAppQuery}, @@ -59,7 +59,7 @@ use windmill_common::{ users::username_to_permissioned_as, utils::{ http_get_from_hub, not_found_if_none, paginate, query_elems_from_hub, require_admin, - Pagination, StripPath, + Pagination, RunnableKind, StripPath, }, variables::{build_crypt, build_crypt_with_key_suffix, encrypt}, worker::{to_raw_value, CLOUD_HOSTED}, diff --git a/backend/windmill-api/src/apps_ee.rs b/backend/windmill-api/src/apps_ee.rs deleted file mode 100644 index a7737664b9..0000000000 --- a/backend/windmill-api/src/apps_ee.rs +++ /dev/null @@ -1,5 +0,0 @@ -use axum::Router; - -pub fn global_unauthed_service() -> Router { - Router::new() -} diff --git a/backend/windmill-api/src/apps_oss.rs b/backend/windmill-api/src/apps_oss.rs new file mode 100644 index 0000000000..346fff9ff1 --- /dev/null +++ b/backend/windmill-api/src/apps_oss.rs @@ -0,0 +1,11 @@ +#[cfg(feature = "private")] +#[allow(unused)] +pub use crate::apps_ee::*; + +#[cfg(not(feature = "private"))] +use axum::Router; + +#[cfg(not(feature = "private"))] +pub fn global_unauthed_service() -> Router { + Router::new() +} diff --git a/backend/windmill-api/src/args.rs b/backend/windmill-api/src/args.rs index e3a2adb38f..0b8f4d5210 100644 --- a/backend/windmill-api/src/args.rs +++ b/backend/windmill-api/src/args.rs @@ -85,7 +85,7 @@ impl RawWebhookArgs { db: &DB, w_id: &str, ) -> Result>, Error> { - use crate::job_helpers_ee::{ + use crate::job_helpers_oss::{ get_random_file_name, get_workspace_s3_resource, upload_file_internal, }; use futures::TryStreamExt; diff --git a/backend/windmill-api/src/audit.rs b/backend/windmill-api/src/audit.rs index 47ba881014..336fd32881 100644 --- a/backend/windmill-api/src/audit.rs +++ b/backend/windmill-api/src/audit.rs @@ -28,7 +28,7 @@ async fn get_audit( Path((w_id, id)): Path<(String, i32)>, ) -> JsonResult { let tx = user_db.begin(&authed).await?; - let audit = windmill_audit::audit_ee::get_audit(tx, id, &w_id).await?; + let audit = windmill_audit::audit_oss::get_audit(tx, id, &w_id).await?; Ok(Json(audit)) } async fn list_audit( @@ -39,6 +39,6 @@ async fn list_audit( Query(lq): Query, ) -> JsonResult> { let tx = user_db.begin(&authed).await?; - let rows = windmill_audit::audit_ee::list_audit(tx, w_id, pagination, lq).await?; + let rows = windmill_audit::audit_oss::list_audit(tx, w_id, pagination, lq).await?; Ok(Json(rows)) } diff --git a/backend/windmill-api/src/auth.rs b/backend/windmill-api/src/auth.rs index 2936b74957..954fcad43c 100644 --- a/backend/windmill-api/src/auth.rs +++ b/backend/windmill-api/src/auth.rs @@ -1,5 +1,5 @@ #[cfg(feature = "enterprise")] -use crate::ee::ExternalJwks; +use crate::ee_oss::ExternalJwks; use axum::{ async_trait, extract::{FromRequestParts, OriginalUri, Query}, @@ -26,6 +26,21 @@ use windmill_common::{ users::{COOKIE_NAME, SUPERADMIN_SECRET_EMAIL}, }; +lazy_static::lazy_static! { + // Global auth cache accessible from main.rs for direct invalidation + pub static ref AUTH_CACHE: Cache<(String, String), ExpiringAuthCache> = Cache::new(300); +} + +// Global function to invalidate a specific token from cache +pub fn invalidate_token_from_cache(token: &str) { + // Remove all cache entries for this token (across all workspaces) + AUTH_CACHE.retain(|(_workspace_id, cached_token), _cached_value| cached_token != token); + tracing::info!( + "Invalidated token from auth cache: {}...", + &token[..token.len().min(8)] + ); +} + #[derive(Clone)] pub struct ExpiringAuthCache { pub authed: ApiAuthed, @@ -33,7 +48,6 @@ pub struct ExpiringAuthCache { } pub struct AuthCache { - cache: Cache<(String, String), ExpiringAuthCache>, db: DB, superadmin_secret: Option, #[cfg(feature = "enterprise")] @@ -47,7 +61,6 @@ impl AuthCache { #[cfg(feature = "enterprise")] ext_jwks: Option>>, ) -> Self { AuthCache { - cache: Cache::new(300), db, superadmin_secret, #[cfg(feature = "enterprise")] @@ -56,7 +69,7 @@ impl AuthCache { } pub async fn invalidate(&self, w_id: &str, token: String) { - self.cache.remove(&(w_id.to_string(), token)); + AUTH_CACHE.remove(&(w_id.to_string(), token)); } pub async fn get_authed(&self, w_id: Option, token: &str) -> Option { @@ -64,14 +77,14 @@ impl AuthCache { w_id.as_ref().unwrap_or(&"".to_string()).to_string(), token.to_string(), ); - let s = self.cache.get(&key).map(|c| c.to_owned()); + let s = AUTH_CACHE.get(&key).map(|c| c.to_owned()); match s { Some(ExpiringAuthCache { authed, expiry }) if expiry > chrono::Utc::now() => { Some(authed) } #[cfg(feature = "enterprise")] _ if token.starts_with("jwt_ext_") => { - let authed_and_exp = match crate::ee::jwt_ext_auth( + let authed_and_exp = match crate::ee_oss::jwt_ext_auth( w_id.as_ref(), token.trim_start_matches("jwt_ext_"), self.ext_jwks.clone(), @@ -86,7 +99,7 @@ impl AuthCache { }; if let Some((authed, exp)) = authed_and_exp.clone() { - self.cache.insert( + AUTH_CACHE.insert( key, ExpiringAuthCache { authed: authed.clone(), @@ -123,7 +136,7 @@ impl AuthCache { username_override, }; - self.cache.insert( + AUTH_CACHE.insert( key, ExpiringAuthCache { authed: authed.clone(), @@ -317,7 +330,7 @@ impl AuthCache { } }; if let Some(authed) = authed_o.as_ref() { - self.cache.insert( + AUTH_CACHE.insert( key, ExpiringAuthCache { authed: authed.clone(), @@ -470,6 +483,27 @@ where } } +fn maybe_get_workspace_id_from_path(path_vec: &[&str]) -> Option { + let workspace_id = if path_vec.len() >= 4 && path_vec[0] == "" && path_vec[2] == "w" { + Some(path_vec[3].to_owned()) + } else if path_vec.len() >= 5 + && path_vec[0] == "" + && path_vec[1] == "api" + && path_vec[2] == "mcp" + && path_vec[3] == "w" + { + Some(path_vec[4].to_owned()) + } else { + if path_vec.len() >= 5 && path_vec[0] == "" && path_vec[2] == "srch" && path_vec[3] == "w" { + Some(path_vec[4].to_owned()) + } else { + None + } + }; + + workspace_id +} + #[async_trait] impl FromRequestParts for ApiAuthed where @@ -482,85 +516,62 @@ where state: &S, ) -> std::result::Result { if parts.method == http::Method::OPTIONS { - return Ok(ApiAuthed { - email: "".to_owned(), - username: "".to_owned(), - is_admin: false, - is_operator: false, - groups: Vec::new(), - folders: Vec::new(), - scopes: None, - username_override: None, - }); + return Ok(ApiAuthed::default()); }; let already_authed = parts.extensions.get::(); - if let Some(authed) = already_authed { - Ok(authed.clone()) - } else { - let already_tokened = parts.extensions.get::(); - let token_o = if let Some(token) = already_tokened { - Some(token.token.clone()) - } else { - extract_token(parts, state).await - }; - let original_uri = OriginalUri::from_request_parts(parts, state) - .await - .ok() - .map(|x| x.0) - .unwrap_or_default(); - let path_vec: Vec<&str> = original_uri.path().split("/").collect(); - let workspace_id = if path_vec.len() >= 4 && path_vec[0] == "" && path_vec[2] == "w" { - Some(path_vec[3].to_owned()) - } else if path_vec.len() >= 5 - && path_vec[0] == "" - && path_vec[1] == "api" - && path_vec[2] == "mcp" - && path_vec[3] == "w" - { - Some(path_vec[4].to_string()) - } else { - if path_vec.len() >= 5 - && path_vec[0] == "" - && path_vec[2] == "srch" - && path_vec[3] == "w" - { - Some(path_vec[4].to_string()) - } else { - None - } - }; - if let Some(token) = token_o { - if let Ok(Extension(cache)) = - Extension::>::from_request_parts(parts, state).await - { - if let Some(authed) = cache.get_authed(workspace_id.clone(), &token).await { - parts.extensions.insert(authed.clone()); - if authed.scopes.as_ref().is_some_and(|scopes| { - scopes - .iter() - .any(|s| s.starts_with("jobs:") || s.starts_with("run:")) - }) && (path_vec.len() < 3 - || (path_vec[4] != "jobs" && path_vec[4] != "jobs_u")) - { - BRUTE_FORCE_COUNTER.increment().await; - return Err(( - StatusCode::UNAUTHORIZED, - format!("Unauthorized scoped token: {:?}", authed.scopes), - )); - } - Span::current().record("username", &authed.username.as_str()); - Span::current().record("email", &authed.email); - if let Some(workspace_id) = workspace_id { - Span::current().record("workspace_id", &workspace_id); - } - return Ok(authed); + if let Some(authed) = already_authed { + return Ok(authed.clone()); + } + + let already_tokened = parts.extensions.get::(); + let token_o = if let Some(token) = already_tokened { + Some(token.token.clone()) + } else { + extract_token(parts, state).await + }; + + if let Some(token) = token_o { + if let Ok(Extension(cache)) = + Extension::>::from_request_parts(parts, state).await + { + let original_uri = OriginalUri::from_request_parts(parts, state) + .await + .ok() + .map(|x| x.0) + .unwrap_or_default(); + let path_vec: Vec<&str> = original_uri.path().split("/").collect(); + let workspace_id = maybe_get_workspace_id_from_path(&path_vec); + + if let Some(authed) = cache.get_authed(workspace_id.clone(), &token).await { + if authed.scopes.as_ref().is_some_and(|scopes| { + scopes + .iter() + .any(|s| s.starts_with("jobs:") || s.starts_with("run:")) + }) && (path_vec.len() < 3 + || (path_vec[4] != "jobs" && path_vec[4] != "jobs_u")) + { + BRUTE_FORCE_COUNTER.increment().await; + return Err(( + StatusCode::UNAUTHORIZED, + format!("Unauthorized scoped token: {:?}", authed.scopes), + )); } + + parts.extensions.insert(authed.clone()); + + Span::current().record("username", &authed.username.as_str()); + Span::current().record("email", &authed.email); + + if let Some(workspace_id) = workspace_id { + Span::current().record("workspace_id", &workspace_id); + } + return Ok(authed); } } - BRUTE_FORCE_COUNTER.increment().await; - Err((StatusCode::UNAUTHORIZED, "Unauthorized".to_owned())) } + BRUTE_FORCE_COUNTER.increment().await; + Err((StatusCode::UNAUTHORIZED, "Unauthorized".to_owned())) } } diff --git a/backend/windmill-api/src/capture.rs b/backend/windmill-api/src/capture.rs index 3164ebe43f..54ad2c35d4 100644 --- a/backend/windmill-api/src/capture.rs +++ b/backend/windmill-api/src/capture.rs @@ -15,7 +15,7 @@ use { #[cfg(all(feature = "enterprise", feature = "gcp_trigger"))] use { - crate::gcp_triggers_ee::{ + crate::gcp_triggers_oss::{ manage_google_subscription, process_google_push_request, validate_jwt_token, CreateUpdateConfig, SubscriptionMode, }, @@ -46,28 +46,24 @@ use serde::de::DeserializeOwned; use windmill_common::error::Error; #[cfg(all(feature = "enterprise", feature = "kafka"))] -use crate::kafka_triggers_ee::KafkaTriggerConfigConnection; +use crate::kafka_triggers_oss::KafkaTriggerConfigConnection; #[cfg(feature = "mqtt_trigger")] use crate::mqtt_triggers::{MqttClientVersion, MqttV3Config, MqttV5Config, SubscribeTopic}; #[cfg(all(feature = "enterprise", feature = "nats"))] -use crate::nats_triggers_ee::NatsTriggerConfigConnection; +use crate::nats_triggers_oss::NatsTriggerConfigConnection; #[cfg(feature = "postgres_trigger")] -use { - crate::postgres_triggers::{ - create_logical_replication_slot, create_pg_publication, generate_random_string, - get_pg_connection, PublicationData, - }, - sqlx::Connection, +use crate::postgres_triggers::{ + create_logical_replication_slot, create_pg_publication, generate_random_string, + get_default_pg_connection, PublicationData, }; use crate::{ args::RawWebhookArgs, db::{ApiAuthed, DB}, users::fetch_api_authed, - utils::RunnableKind, }; use axum::{ @@ -85,7 +81,7 @@ use windmill_common::{ db::UserDB, error::{JsonResult, Result}, triggers::{RunnableFormat, RunnableFormatVersion, TriggerKind}, - utils::{not_found_if_none, paginate, Pagination, StripPath}, + utils::{not_found_if_none, paginate, Pagination, RunnableKind, StripPath}, worker::{to_raw_value, CLOUD_HOSTED}, }; @@ -304,13 +300,15 @@ async fn set_postgres_trigger_config( user_db: UserDB, mut capture_config: NewCaptureConfig, ) -> Result { + use windmill_common::error::to_anyhow; + let Some(TriggerConfig::Postgres(postgres_config)) = capture_config.trigger_config.as_mut() else { return Err(Error::BadRequest("Invalid postgres config".to_string())); }; if postgres_config.basic_mode.unwrap_or(false) { - let mut pg_connection = get_pg_connection( + let mut pg_connection = get_default_pg_connection( authed, Some(user_db), &db, @@ -319,22 +317,26 @@ async fn set_postgres_trigger_config( ) .await?; - let mut tx = pg_connection.begin().await?; + let tx = pg_connection.transaction().await.map_err(to_anyhow)?; let publication_name = format!("windmill_capture_{}", generate_random_string()); let replication_slot_name = publication_name.clone(); - create_logical_replication_slot(&mut tx, &replication_slot_name).await?; + create_logical_replication_slot(tx.client(), &replication_slot_name) + .await + .map_err(to_anyhow)?; create_pg_publication( - &mut tx, + tx.client(), &publication_name, postgres_config.publication.table_to_track.as_deref(), &postgres_config.publication.transaction_to_track, ) - .await?; + .await + .map_err(to_anyhow)?; + + tx.commit().await.map_err(to_anyhow)?; - tx.commit().await?; postgres_config.publication_name = Some(publication_name); postgres_config.replication_slot_name = Some(replication_slot_name); } else { @@ -905,7 +907,7 @@ async fn gcp_payload( headers: HeaderMap, request: Request, ) -> Result { - use crate::{gcp_triggers_ee::GcpTrigger, trigger_helpers::TriggerJobArgs}; + use crate::{gcp_triggers_oss::GcpTrigger, trigger_helpers::TriggerJobArgs}; let is_flow = matches!(runnable_kind, RunnableKind::Flow); let (gcp_trigger_config, owner, email): (GcpTriggerConfig, _, _) = diff --git a/backend/windmill-api/src/configs.rs b/backend/windmill-api/src/configs.rs index e8770b2b5e..9110d104c8 100644 --- a/backend/windmill-api/src/configs.rs +++ b/backend/windmill-api/src/configs.rs @@ -14,7 +14,7 @@ use axum::{ use serde::{Deserialize, Serialize}; use sqlx::FromRow; -use windmill_audit::audit_ee::audit_log; +use windmill_audit::audit_oss::audit_log; use windmill_audit::ActionKind; use windmill_common::{ error::{self}, diff --git a/backend/windmill-api/src/db.rs b/backend/windmill-api/src/db.rs index 85f1ee485f..b70532a8a9 100644 --- a/backend/windmill-api/src/db.rs +++ b/backend/windmill-api/src/db.rs @@ -16,7 +16,7 @@ use sqlx::{ }; use tokio::task::JoinHandle; -use windmill_audit::audit_ee::{AuditAuthor, AuditAuthorable}; +use windmill_audit::audit_oss::{AuditAuthor, AuditAuthorable}; use windmill_common::{ db::{Authable, Authed}, error::Error, @@ -804,10 +804,18 @@ async fn fix_job_completed_index(db: &DB) -> Result<(), Error> { .execute(db) .await?; }); + + run_windmill_migration!("v2_job_queue_suspend", db, |tx| { + sqlx::query!( + "CREATE INDEX CONCURRENTLY IF NOT EXISTS v2_job_queue_suspend ON v2_job_queue (workspace_id, suspend) WHERE suspend > 0;" + ) + .execute(db) + .await?; + }); Ok(()) } -#[derive(Clone, Debug, Hash, Eq, PartialEq)] +#[derive(Clone, Debug, Default, Hash, Eq, PartialEq)] pub struct ApiAuthed { pub email: String, pub username: String, diff --git a/backend/windmill-api/src/ee.rs b/backend/windmill-api/src/ee_oss.rs similarity index 61% rename from backend/windmill-api/src/ee.rs rename to backend/windmill-api/src/ee_oss.rs index cddb639e95..3fb1271a73 100644 --- a/backend/windmill-api/src/ee.rs +++ b/backend/windmill-api/src/ee_oss.rs @@ -1,15 +1,21 @@ +#[cfg(feature = "private")] +#[allow(unused)] +pub use crate::ee::*; + +#[cfg(not(feature = "private"))] use anyhow::anyhow; -#[cfg(feature = "enterprise")] +#[cfg(all(feature = "enterprise", not(feature = "private")))] use std::sync::Arc; -#[cfg(feature = "enterprise")] +#[cfg(all(feature = "enterprise", not(feature = "private")))] use tokio::sync::RwLock; +#[cfg(not(feature = "private"))] pub async fn validate_license_key(_license_key: String) -> anyhow::Result<(String, bool)> { // Implementation is not open source Err(anyhow!("License can't be validated in Windmill CE")) } -#[cfg(feature = "enterprise")] +#[cfg(all(feature = "enterprise", not(feature = "private")))] pub async fn jwt_ext_auth( _w_id: Option<&String>, _token: &str, @@ -20,10 +26,10 @@ pub async fn jwt_ext_auth( Err(anyhow!("External JWT auth is not open source")) } -#[cfg(feature = "enterprise")] +#[cfg(all(feature = "enterprise", not(feature = "private")))] pub struct ExternalJwks; -#[cfg(feature = "enterprise")] +#[cfg(all(feature = "enterprise", not(feature = "private")))] impl ExternalJwks { pub async fn load() -> Option>> { // Implementation is not open source diff --git a/backend/windmill-api/src/flows.rs b/backend/windmill-api/src/flows.rs index e34b2cacaf..e8ebe6ccc5 100644 --- a/backend/windmill-api/src/flows.rs +++ b/backend/windmill-api/src/flows.rs @@ -12,7 +12,7 @@ use crate::db::ApiAuthed; use crate::triggers::{ get_triggers_count_internal, list_tokens_internal, TriggersCount, TruncatedTokenWithEmail, }; -use crate::utils::{RunnableKind, WithStarredInfoQuery}; +use crate::utils::WithStarredInfoQuery; use crate::{ db::DB, schedule::clear_schedule, @@ -31,7 +31,7 @@ use hyper::StatusCode; use serde::{Deserialize, Serialize}; use sql_builder::prelude::*; use sqlx::{FromRow, Postgres, Transaction}; -use windmill_audit::audit_ee::audit_log; +use windmill_audit::audit_oss::audit_log; use windmill_audit::ActionKind; use windmill_common::utils::query_elems_from_hub; use windmill_common::worker::to_raw_value; @@ -43,7 +43,7 @@ use windmill_common::{ jobs::JobPayload, schedule::Schedule, scripts::Schema, - utils::{http_get_from_hub, not_found_if_none, paginate, Pagination, StripPath}, + utils::{http_get_from_hub, not_found_if_none, paginate, Pagination, RunnableKind, StripPath}, }; use windmill_git_sync::{handle_deployment_metadata, DeployedObject}; use windmill_queue::{push, schedule::push_scheduled_job, PushIsolationLevel}; @@ -477,7 +477,7 @@ async fn create_flow( false, None, true, - nf.tag, + None, None, None, None, diff --git a/backend/windmill-api/src/folders.rs b/backend/windmill-api/src/folders.rs index 5047a96cad..1e4d2e09c0 100644 --- a/backend/windmill-api/src/folders.rs +++ b/backend/windmill-api/src/folders.rs @@ -23,7 +23,7 @@ use axum::{ }; use lazy_static::lazy_static; use regex::Regex; -use windmill_audit::audit_ee::audit_log; +use windmill_audit::audit_oss::audit_log; use windmill_audit::ActionKind; use windmill_common::{ db::UserDB, diff --git a/backend/windmill-api/src/gcp_triggers_ee.rs b/backend/windmill-api/src/gcp_triggers_oss.rs similarity index 76% rename from backend/windmill-api/src/gcp_triggers_ee.rs rename to backend/windmill-api/src/gcp_triggers_oss.rs index 0dc672580c..78005309bc 100644 --- a/backend/windmill-api/src/gcp_triggers_ee.rs +++ b/backend/windmill-api/src/gcp_triggers_oss.rs @@ -1,29 +1,38 @@ -use crate::db::{ApiAuthed, DB}; -use crate::trigger_helpers::TriggerJobArgs; -use axum::{extract::Request, Router}; -use http::HeaderMap; -use serde::{Deserialize, Serialize}; -use serde_json::value::RawValue; -use sqlx::prelude::FromRow; -use sqlx::types::Json as SqlxJson; -use std::collections::HashMap; -use windmill_common::db::UserDB; -use windmill_common::worker::to_raw_value; -use windmill_common::{ - error::{Error as WindmillError, Result as WindmillResult}, - triggers::TriggerKind, - utils::empty_as_none, +#[cfg(feature = "private")] +#[allow(unused)] +pub use crate::gcp_triggers_ee::*; + +#[cfg(not(feature = "private"))] +use { + crate::db::{ApiAuthed, DB}, + crate::trigger_helpers::TriggerJobArgs, + axum::{extract::Request, Router}, + http::HeaderMap, + serde::{Deserialize, Serialize}, + serde_json::value::RawValue, + sqlx::prelude::FromRow, + sqlx::types::Json as SqlxJson, + std::collections::HashMap, + windmill_common::db::UserDB, + windmill_common::worker::to_raw_value, + windmill_common::{ + error::{Error as WindmillError, Result as WindmillResult}, + triggers::TriggerKind, + utils::empty_as_none, + }, }; #[derive(sqlx::Type, Debug, Deserialize, Serialize)] #[serde(rename_all(serialize = "lowercase", deserialize = "lowercase"))] #[sqlx(type_name = "DELIVERY_MODE", rename_all = "lowercase")] #[allow(unused)] +#[cfg(not(feature = "private"))] pub enum DeliveryType { Pull, Push, } +#[cfg(not(feature = "private"))] impl Default for DeliveryType { fn default() -> Self { Self::Pull @@ -32,6 +41,7 @@ impl Default for DeliveryType { #[derive(FromRow, Deserialize, Serialize, Debug)] #[allow(unused)] +#[cfg(not(feature = "private"))] pub struct PushConfig { #[serde(deserialize_with = "empty_as_none")] route_path: Option, @@ -42,6 +52,7 @@ pub struct PushConfig { } #[derive(Default, Debug, Serialize, Deserialize)] #[allow(unused)] +#[cfg(not(feature = "private"))] pub struct CreateUpdateConfig { pub delivery_type: DeliveryType, #[serde(default, deserialize_with = "empty_as_none")] @@ -50,6 +61,7 @@ pub struct CreateUpdateConfig { } #[derive(Debug, Deserialize, Serialize)] +#[cfg(not(feature = "private"))] pub struct ExistingGcpSubscription { pub subscription_id: String, pub base_endpoint: String, @@ -58,15 +70,18 @@ pub struct ExistingGcpSubscription { #[derive(Debug, Deserialize, Serialize, sqlx::Type)] #[serde(rename_all = "snake_case")] #[sqlx(type_name = "GCP_SUBSCRIPTION_MODE", rename_all = "snake_case")] +#[cfg(not(feature = "private"))] pub enum SubscriptionMode { Existing, CreateUpdate, } +#[cfg(not(feature = "private"))] pub fn workspaced_service() -> Router { Router::new() } +#[cfg(not(feature = "private"))] pub fn start_consuming_gcp_pubsub_event( _db: DB, mut _killpill_rx: tokio::sync::broadcast::Receiver<()>, @@ -74,6 +89,7 @@ pub fn start_consuming_gcp_pubsub_event( // implementation is not open source } +#[cfg(not(feature = "private"))] pub async fn manage_google_subscription( _authed: ApiAuthed, _db: &DB, @@ -91,6 +107,7 @@ pub async fn manage_google_subscription( Ok(CreateUpdateConfig::default()) } +#[cfg(not(feature = "private"))] pub async fn process_google_push_request( _headers: HeaderMap, _request: Request, @@ -98,6 +115,7 @@ pub async fn process_google_push_request( Ok((String::new(), HashMap::new())) } +#[cfg(not(feature = "private"))] pub async fn validate_jwt_token( _db: &DB, _user_db: UserDB, @@ -110,11 +128,13 @@ pub async fn validate_jwt_token( Ok(()) } +#[cfg(not(feature = "private"))] pub fn gcp_push_route_handler() -> Router { Router::new() } #[derive(FromRow, Deserialize, Serialize, Debug)] +#[cfg(not(feature = "private"))] pub struct GcpTrigger { pub gcp_resource_path: String, pub subscription_id: String, @@ -135,7 +155,7 @@ pub struct GcpTrigger { pub last_server_ping: Option>, pub enabled: bool, } - +#[cfg(not(feature = "private"))] impl TriggerJobArgs for GcpTrigger { fn v1_payload_fn(payload: String) -> HashMap> { HashMap::from([("payload".to_string(), to_raw_value(&payload))]) diff --git a/backend/windmill-api/src/git_sync_ee.rs b/backend/windmill-api/src/git_sync_ee.rs deleted file mode 100644 index b72afac189..0000000000 --- a/backend/windmill-api/src/git_sync_ee.rs +++ /dev/null @@ -1,9 +0,0 @@ -use axum::routing::Router; - -pub fn workspaced_service() -> Router { - Router::new() -} - -pub fn global_service() -> Router { - Router::new() -} \ No newline at end of file diff --git a/backend/windmill-api/src/git_sync_oss.rs b/backend/windmill-api/src/git_sync_oss.rs new file mode 100644 index 0000000000..0451d88699 --- /dev/null +++ b/backend/windmill-api/src/git_sync_oss.rs @@ -0,0 +1,16 @@ +#[cfg(feature = "private")] +#[allow(unused)] +pub use crate::git_sync_ee::*; + +#[cfg(not(feature = "private"))] +use axum::routing::Router; + +#[cfg(not(feature = "private"))] +pub fn workspaced_service() -> Router { + Router::new() +} + +#[cfg(not(feature = "private"))] +pub fn global_service() -> Router { + Router::new() +} diff --git a/backend/windmill-api/src/groups.rs b/backend/windmill-api/src/groups.rs index e25d3ee7da..d5da9c25f5 100644 --- a/backend/windmill-api/src/groups.rs +++ b/backend/windmill-api/src/groups.rs @@ -14,7 +14,7 @@ use axum::{ routing::{delete, get, post}, Json, Router, }; -use windmill_audit::audit_ee::audit_log; +use windmill_audit::audit_oss::audit_log; use windmill_audit::ActionKind; use windmill_common::worker::CLOUD_HOSTED; use windmill_common::{ diff --git a/backend/windmill-api/src/http_trigger_args.rs b/backend/windmill-api/src/http_trigger_args.rs index 6aab75e46e..8fb96d42f0 100644 --- a/backend/windmill-api/src/http_trigger_args.rs +++ b/backend/windmill-api/src/http_trigger_args.rs @@ -21,7 +21,7 @@ use crate::{ pub struct RawHttpTriggerArgs(pub RawWebhookArgs); -#[derive(Serialize, Deserialize, sqlx::Type, Debug, Clone, Hash, Eq, PartialEq)] +#[derive(Serialize, Deserialize, sqlx::Type, Debug, Clone, Copy, Hash, Eq, PartialEq)] #[sqlx(type_name = "HTTP_METHOD", rename_all = "lowercase")] #[serde(rename_all = "lowercase")] pub enum HttpMethod { diff --git a/backend/windmill-api/src/http_trigger_auth.rs b/backend/windmill-api/src/http_trigger_auth.rs index c197dcc9b0..85288d87b8 100644 --- a/backend/windmill-api/src/http_trigger_auth.rs +++ b/backend/windmill-api/src/http_trigger_auth.rs @@ -441,8 +441,8 @@ pub struct BasicAuthAuthentication { #[derive(Debug, Serialize, Deserialize, Clone)] pub struct ApiKeyAuthentication { - api_key_header: String, - api_key_secret: String, + pub api_key_header: String, + pub api_key_secret: String, } #[derive(Debug, Eq, PartialEq, Hash, Clone, Copy, Serialize, Deserialize)] diff --git a/backend/windmill-api/src/http_triggers.rs b/backend/windmill-api/src/http_triggers.rs index b17c8205ad..a42bdcac53 100644 --- a/backend/windmill-api/src/http_triggers.rs +++ b/backend/windmill-api/src/http_triggers.rs @@ -1,7 +1,7 @@ #[cfg(feature = "http_trigger")] use crate::http_trigger_args::{HttpMethod, RawHttpTriggerArgs}; #[cfg(feature = "parquet")] -use crate::job_helpers_ee::get_workspace_s3_resource; +use crate::job_helpers_oss::get_workspace_s3_resource; use crate::resources::try_get_resource_from_db_as; use crate::trigger_helpers::{get_runnable_format, RunnableId}; use crate::utils::{non_empty_str, ExpiringCacheEntry}; @@ -14,6 +14,7 @@ use crate::{ }, users::fetch_api_authed, }; +use anyhow::anyhow; use axum::response::Response; use axum::{ extract::{Path, Query}, @@ -28,13 +29,14 @@ use quick_cache::sync::Cache; use serde::{Deserialize, Serialize}; use sql_builder::{bind::Bind, SqlBuilder}; use sqlx::prelude::FromRow; -use sqlx::PgTransaction; +use sqlx::PgConnection; use std::borrow::Cow; +use std::collections::HashSet; use std::{collections::HashMap, sync::Arc}; use tokio::sync::{RwLock, RwLockReadGuard}; use tower_http::cors::CorsLayer; -use windmill_audit::{audit_ee::audit_log, ActionKind}; -use windmill_common::error::Error; +use windmill_audit::{audit_oss::audit_log, ActionKind}; +use windmill_common::error::{Error, Result as WindmillResult}; #[cfg(feature = "parquet")] use windmill_common::s3_helpers::build_object_store_client; use windmill_common::{ @@ -42,7 +44,7 @@ use windmill_common::{ error::{self, JsonResult}, s3_helpers::S3Object, triggers::TriggerKind, - utils::{not_found_if_none, paginate, require_admin, Pagination, StripPath}, + utils::{empty_as_none, not_found_if_none, paginate, require_admin, Pagination, StripPath}, worker::CLOUD_HOSTED, }; use windmill_git_sync::handle_deployment_metadata; @@ -79,6 +81,7 @@ pub fn routes_global_service() -> Router { pub fn workspaced_service() -> Router { Router::new() .route("/create", post(create_trigger)) + .route("/create_many", post(create_many_http_trigger)) .route("/list", get(list_triggers)) .route("/get/*path", get(get_trigger)) .route("/update/*path", post(update_trigger)) @@ -111,6 +114,8 @@ struct NewTrigger { static_asset_config: Option>, http_method: HttpMethod, workspaced_route: Option, + summary: Option, + description: Option, is_static_website: bool, wrap_body: Option, raw_string: Option, @@ -131,6 +136,8 @@ pub struct HttpTrigger { pub is_async: bool, pub authentication_method: AuthenticationMethod, pub http_method: HttpMethod, + pub summary: Option, + pub description: Option, #[serde(skip_serializing_if = "Option::is_none")] pub static_asset_config: Option>, pub is_static_website: bool, @@ -150,6 +157,8 @@ struct EditTrigger { authentication_method: AuthenticationMethod, #[serde(deserialize_with = "non_empty_str")] authentication_resource_path: Option, + summary: Option, + description: Option, http_method: HttpMethod, static_asset_config: Option>, workspaced_route: Option, @@ -164,6 +173,7 @@ pub struct ListTriggerQuery { pub per_page: Option, pub path: Option, pub is_flow: Option, + #[serde(default, deserialize_with = "empty_as_none")] pub path_start: Option, } @@ -185,6 +195,8 @@ async fn list_triggers( "wrap_body", "raw_string", "script_path", + "summary", + "description", "is_flow", "http_method", "edited_by", @@ -239,6 +251,8 @@ async fn get_trigger( route_path_key, workspaced_route, script_path, + summary, + description, is_flow, http_method as "http_method: _", edited_by, @@ -273,7 +287,7 @@ async fn get_trigger( fn validate_authentication_method( authentication_method: AuthenticationMethod, raw_string: Option, -) -> error::Result<()> { +) -> WindmillResult<()> { match (authentication_method, raw_string) { (AuthenticationMethod::CustomScript, raw) if !raw.unwrap_or(false) == true => { return Err(Error::BadRequest( @@ -287,57 +301,21 @@ fn validate_authentication_method( Ok(()) } -async fn increase_trigger_version_and_commit(mut tx: PgTransaction<'_>) -> error::Result<()> { +async fn increase_trigger_version(tx: &mut PgConnection) -> WindmillResult<()> { sqlx::query!("SELECT nextval('http_trigger_version_seq')",) - .fetch_one(&mut *tx) + .fetch_one(tx) .await?; - tx.commit().await?; - Ok(()) } -async fn create_trigger( - authed: ApiAuthed, - Extension(db): Extension, - Extension(user_db): Extension, - Path(w_id): Path, - Json(ct): Json, -) -> error::Result<(StatusCode, String)> { - require_admin(authed.is_admin, &authed.username)?; - - if !VALID_ROUTE_PATH_RE.is_match(&ct.route_path) { - return Err(error::Error::BadRequest("Invalid route path".to_string())); - } - - validate_authentication_method(ct.authentication_method, ct.raw_string)?; - - // route path key is extracted from the route path to check for uniqueness - // it replaces /?:{key} with :key - // it will also remove the leading / if present, not an issue as we only allow : after slashes - let route_path_key = ROUTE_PATH_KEY_RE.replace_all(&ct.route_path, ":key"); - let exists = route_path_key_exists( - &route_path_key, - &ct.http_method, - &w_id, - None, - ct.workspaced_route, - &db, - ) - .await?; - if exists { - return Err(error::Error::BadRequest( - "A route already exists with this path".to_string(), - )); - } - - if *CLOUD_HOSTED && (ct.is_static_website || ct.static_asset_config.is_some()) { - return Err(error::Error::BadRequest( - "Static website and static asset are not supported on cloud".to_string(), - )); - } - - let mut tx = user_db.begin(&authed).await?; +async fn create_trigger_inner( + tx: &mut PgConnection, + w_id: &str, + authed: &ApiAuthed, + new_http_trigger: &NewTrigger, + route_path_key: &str, +) -> WindmillResult<()> { sqlx::query!( r#" INSERT INTO http_trigger ( @@ -350,6 +328,8 @@ async fn create_trigger( wrap_body, raw_string, script_path, + summary, + description, is_flow, is_async, authentication_method, @@ -361,55 +341,209 @@ async fn create_trigger( is_static_website ) VALUES ( - $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, now(), $17 + $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, now(), $19 ) "#, w_id, - ct.path, - ct.route_path, + new_http_trigger.path, + new_http_trigger.route_path, &route_path_key, - ct.workspaced_route, - ct.authentication_resource_path, - ct.wrap_body.unwrap_or(false), - ct.raw_string.unwrap_or(false), - ct.script_path, - ct.is_flow, - ct.is_async, - ct.authentication_method as _, - ct.http_method as _, - ct.static_asset_config as _, + new_http_trigger.workspaced_route, + new_http_trigger.authentication_resource_path, + new_http_trigger.wrap_body.unwrap_or(false), + new_http_trigger.raw_string.unwrap_or(false), + new_http_trigger.script_path, + new_http_trigger.summary, + new_http_trigger.description, + new_http_trigger.is_flow, + new_http_trigger.is_async, + new_http_trigger.authentication_method as _, + new_http_trigger.http_method as _, + new_http_trigger.static_asset_config as _, &authed.username, &authed.email, - ct.is_static_website + new_http_trigger.is_static_website ) .execute(&mut *tx) .await?; audit_log( &mut *tx, - &authed, + authed, "http_triggers.create", ActionKind::Create, &w_id, - Some(ct.path.as_str()), + Some(new_http_trigger.path.as_str()), None, ) .await?; - increase_trigger_version_and_commit(tx).await?; + increase_trigger_version(tx).await?; + + Ok(()) +} + +fn check_no_duplicates<'trigger>( + new_http_triggers: &[NewTrigger], + route_path_key: &[Cow<'trigger, str>], +) -> Result<(), Error> { + let mut seen = HashSet::with_capacity(new_http_triggers.len()); + + for (i, trigger) in new_http_triggers.iter().enumerate() { + if !seen.insert(( + &route_path_key[i], + trigger.http_method, + trigger.workspaced_route, + )) { + return Err(Error::BadRequest(format!( + "Duplicate HTTP route detected: '{}'. Each HTTP route must have a unique 'route_path'.", + &trigger.route_path + ))); + } + } + + Ok(()) +} + +async fn create_many_http_trigger( + authed: ApiAuthed, + Extension(db): Extension, + Extension(user_db): Extension, + Path(w_id): Path, + Json(new_http_triggers): Json>, +) -> WindmillResult<(StatusCode, String)> { + require_admin(authed.is_admin, &authed.username)?; + + let error_wrapper = |path: &str, error: Error| -> Error { + anyhow!( + "Error occurred for HTTP route at route path: {}, error: {}", + path, + error + ) + .into() + }; + + let mut route_path_keys = Vec::with_capacity(new_http_triggers.len()); + + for new_http_trigger in new_http_triggers.iter() { + let route_path_key = validate_http_trigger(&db, &w_id, new_http_trigger) + .await + .map_err(|err| error_wrapper(&new_http_trigger.route_path, err))?; + + route_path_keys.push(route_path_key); + } + + check_no_duplicates(&new_http_triggers, &route_path_keys)?; + + let mut tx = user_db.begin(&authed).await?; + + for (i, new_http_trigger) in new_http_triggers.iter().enumerate() { + create_trigger_inner( + &mut tx, + &w_id, + &authed, + new_http_trigger, + &route_path_keys[i], + ) + .await + .map_err(|err| error_wrapper(&new_http_trigger.route_path, err))?; + } + + tx.commit().await?; + + for http_trigger in new_http_triggers.into_iter() { + handle_deployment_metadata( + &authed.email, + &authed.username, + &db, + &w_id, + windmill_git_sync::DeployedObject::HttpTrigger { path: http_trigger.path.clone() }, + Some(format!("HTTP route '{}' created", http_trigger.path)), + true, + ) + .await?; + } + + Ok((StatusCode::CREATED, format!("Created all HTTP routes"))) +} + +async fn validate_http_trigger<'trigger>( + db: &DB, + w_id: &str, + new_http_trigger: &'trigger NewTrigger, +) -> WindmillResult> { + if !VALID_ROUTE_PATH_RE.is_match(&new_http_trigger.route_path) { + return Err(error::Error::BadRequest("Invalid route path".to_string())); + } + + validate_authentication_method( + new_http_trigger.authentication_method, + new_http_trigger.raw_string, + )?; + + // route path key is extracted from the route path to check for uniqueness + // it replaces /?:{key} with :key + // it will also remove the leading / if present, not an issue as we only allow : after slashes + let route_path_key = ROUTE_PATH_KEY_RE.replace_all(&new_http_trigger.route_path, ":key"); + + let exists = route_path_key_exists( + &route_path_key, + &new_http_trigger.http_method, + &w_id, + None, + new_http_trigger.workspaced_route, + db, + ) + .await?; + + if exists { + return Err(error::Error::BadRequest( + "A route already exists with this path".to_string(), + )); + } + + if *CLOUD_HOSTED + && (new_http_trigger.is_static_website || new_http_trigger.static_asset_config.is_some()) + { + return Err(error::Error::BadRequest( + "Static website and static asset are not supported on cloud".to_string(), + )); + } + + Ok(route_path_key) +} + +async fn create_trigger( + authed: ApiAuthed, + Extension(db): Extension, + Extension(user_db): Extension, + Path(w_id): Path, + Json(new_http_trigger): Json, +) -> WindmillResult<(StatusCode, String)> { + require_admin(authed.is_admin, &authed.username)?; + + let route_path_key = validate_http_trigger(&db, &w_id, &new_http_trigger).await?; + + let mut tx = user_db.begin(&authed).await?; + + let http_trigger_path = new_http_trigger.path.clone(); + + create_trigger_inner(&mut tx, &w_id, &authed, &new_http_trigger, &route_path_key).await?; + + tx.commit().await?; handle_deployment_metadata( &authed.email, &authed.username, &db, &w_id, - windmill_git_sync::DeployedObject::HttpTrigger { path: ct.path.clone() }, - Some(format!("HTTP trigger '{}' created", ct.path)), + windmill_git_sync::DeployedObject::HttpTrigger { path: new_http_trigger.path.clone() }, + Some(format!("HTTP route '{}' created", new_http_trigger.path)), true, ) .await?; - Ok((StatusCode::CREATED, format!("{}", ct.path))) + Ok((StatusCode::CREATED, format!("{}", http_trigger_path))) } async fn update_trigger( @@ -418,8 +552,9 @@ async fn update_trigger( Extension(db): Extension, Path((w_id, path)): Path<(String, StripPath)>, Json(ct): Json, -) -> error::Result { +) -> WindmillResult { let path = path.to_path(); + if *CLOUD_HOSTED && (ct.is_static_website || ct.static_asset_config.is_some()) { return Err(error::Error::BadRequest( "Static website and static asset are not supported on cloud".to_string(), @@ -478,11 +613,13 @@ async fn update_trigger( email = $13, is_async = $14, authentication_method = $15, + summary = $16, + description = $17, edited_at = now(), - is_static_website = $16 + is_static_website = $18 WHERE - workspace_id = $17 AND - path = $18 + workspace_id = $19 AND + path = $20 "#, route_path, &route_path_key, @@ -499,6 +636,8 @@ async fn update_trigger( &authed.email, ct.is_async, ct.authentication_method as _, + ct.summary, + ct.description, ct.is_static_website, w_id, path, @@ -563,7 +702,9 @@ async fn update_trigger( ) .await?; - increase_trigger_version_and_commit(tx).await?; + increase_trigger_version(&mut tx).await?; + + tx.commit().await?; handle_deployment_metadata( &authed.email, @@ -571,7 +712,7 @@ async fn update_trigger( &db, &w_id, windmill_git_sync::DeployedObject::HttpTrigger { path: ct.path.clone() }, - Some(format!("HTTP trigger '{}' updated", ct.path)), + Some(format!("HTTP route '{}' updated", ct.path)), true, ) .await?; @@ -584,7 +725,7 @@ async fn delete_trigger( Extension(db): Extension, Extension(user_db): Extension, Path((w_id, path)): Path<(String, StripPath)>, -) -> error::Result { +) -> WindmillResult { require_admin(authed.is_admin, &authed.username)?; let path = path.to_path(); let mut tx = user_db.begin(&authed).await?; @@ -609,7 +750,9 @@ async fn delete_trigger( ) .await?; - increase_trigger_version_and_commit(tx).await?; + increase_trigger_version(&mut tx).await?; + + tx.commit().await?; handle_deployment_metadata( &authed.email, @@ -617,12 +760,12 @@ async fn delete_trigger( &db, &w_id, windmill_git_sync::DeployedObject::HttpTrigger { path: path.to_string() }, - Some(format!("HTTP trigger '{}' deleted", path)), + Some(format!("HTTP route '{}' deleted", path)), true, ) .await?; - Ok(format!("HTTP trigger {path} deleted")) + Ok(format!("HTTP route {path} deleted")) } async fn exists_trigger( @@ -660,7 +803,7 @@ async fn route_path_key_exists( trigger_path: Option<&str>, workspaced_route: Option, db: &DB, -) -> error::Result { +) -> WindmillResult { let exists = if *CLOUD_HOSTED { sqlx::query_scalar!( r#" @@ -861,7 +1004,7 @@ pub async fn refresh_routers(db: &DB) -> Result<(bool, RwLockReadGuard<'_, Route .insert(format!("{}/*wm_subpath", full_path), trigger.clone()) .unwrap_or_else(|e| { tracing::warn!( - "Failed to consider http trigger route {}/*wm_subpath: {:?}", + "Failed to consider HTTP route {}/*wm_subpath: {:?}", full_path, e, ); @@ -870,11 +1013,7 @@ pub async fn refresh_routers(db: &DB) -> Result<(bool, RwLockReadGuard<'_, Route router .insert(full_path.clone(), trigger.clone()) .unwrap_or_else(|e| { - tracing::warn!( - "Failed to consider http trigger route {}: {:?}", - full_path, - e, - ); + tracing::warn!("Failed to consider HTTP route {}: {:?}", full_path, e,); }); } @@ -898,7 +1037,7 @@ async fn get_http_route_trigger( db: &DB, user_db: UserDB, method: &http::Method, -) -> error::Result<(TriggerRoute, String, HashMap, ApiAuthed)> { +) -> WindmillResult<(TriggerRoute, String, HashMap, ApiAuthed)> { let http_method: HttpMethod = method.try_into()?; let requested_path = format!("/{}", route_path); @@ -947,11 +1086,11 @@ async fn get_http_route_trigger( ); let exists = match HTTP_ACCESS_CACHE.get(&cache_key) { Some(cache_entry) if cache_entry.expiry > std::time::Instant::now() => { - tracing::debug!("HTTP access cache hit for trigger {}", trigger.path); + tracing::debug!("HTTP access cache hit for route {}", trigger.path); true } _ => { - tracing::debug!("HTTP access cache miss for trigger {}", trigger.path); + tracing::debug!("HTTP access cache miss for route {}", trigger.path); let mut tx = user_db.begin(&authed).await?; let exists = sqlx::query_scalar!( r#" @@ -1002,7 +1141,7 @@ async fn get_http_route_trigger( trigger.email.clone(), &trigger.workspace_id, &db, - Some(username_override.unwrap_or(format!("http-{}", trigger.path))), + Some(username_override.unwrap_or(format!("HTTP-{}", trigger.path))), ) .await?; @@ -1030,6 +1169,14 @@ async fn route_job( .await .map_err(|e| e.into_response())?; + if trigger.script_path.is_empty() && trigger.static_asset_config.is_none() { + return Err(Error::NotFound(format!( + "Runnable path of HTTP route at path: {}", + trigger.path + )) + .into_response()); + } + let args = args .process_args( &authed, @@ -1066,15 +1213,15 @@ async fn route_job( let authentication_method = match HTTP_AUTH_CACHE.get(&cache_key) { Some(cache_entry) if cache_entry.expiry > std::time::Instant::now() => { - tracing::debug!("HTTP auth method cache hit for trigger {}", trigger.path); + tracing::debug!("HTTP auth method cache hit for route {}", trigger.path); cache_entry.value } _ => { - tracing::debug!("HTTP auth method cache miss for trigger {}", trigger.path); + tracing::debug!("HTTP auth method cache miss for route {}", trigger.path); let auth_method = try_get_resource_from_db_as::< crate::http_trigger_auth::AuthenticationMethod, >( - authed.clone(), + &authed, Some(user_db.clone()), &db, &resource_path, diff --git a/backend/windmill-api/src/indexer_ee.rs b/backend/windmill-api/src/indexer_ee.rs deleted file mode 100644 index 2ccca92c27..0000000000 --- a/backend/windmill-api/src/indexer_ee.rs +++ /dev/null @@ -1,9 +0,0 @@ -use axum::Router; - -pub fn workspaced_service() -> Router { - Router::new() -} - -pub fn global_service() -> Router { - Router::new() -} diff --git a/backend/windmill-api/src/indexer_oss.rs b/backend/windmill-api/src/indexer_oss.rs new file mode 100644 index 0000000000..eee87acdcb --- /dev/null +++ b/backend/windmill-api/src/indexer_oss.rs @@ -0,0 +1,16 @@ +#[cfg(feature = "private")] +#[allow(unused)] +pub use crate::indexer_ee::*; + +#[cfg(not(feature = "private"))] +use axum::Router; + +#[cfg(not(feature = "private"))] +pub fn workspaced_service() -> Router { + Router::new() +} + +#[cfg(not(feature = "private"))] +pub fn global_service() -> Router { + Router::new() +} diff --git a/backend/windmill-api/src/inkeep_oss.rs b/backend/windmill-api/src/inkeep_oss.rs new file mode 100644 index 0000000000..ca1ffebd21 --- /dev/null +++ b/backend/windmill-api/src/inkeep_oss.rs @@ -0,0 +1,11 @@ +#[cfg(feature = "private")] +#[allow(unused)] +pub use crate::inkeep_ee::*; + +#[cfg(not(feature = "private"))] +use axum::Router; + +#[cfg(not(feature = "private"))] +pub fn global_service() -> Router { + Router::new() +} diff --git a/backend/windmill-api/src/job_helpers_ee.rs b/backend/windmill-api/src/job_helpers_oss.rs similarity index 70% rename from backend/windmill-api/src/job_helpers_ee.rs rename to backend/windmill-api/src/job_helpers_oss.rs index 4e14d16c5f..53d4a796a1 100644 --- a/backend/windmill-api/src/job_helpers_ee.rs +++ b/backend/windmill-api/src/job_helpers_oss.rs @@ -1,34 +1,45 @@ +#[cfg(feature = "private")] +#[allow(unused)] +pub use crate::job_helpers_ee::*; + +#[cfg(not(feature = "private"))] use axum::Router; +#[cfg(not(feature = "private"))] use serde::Serialize; +#[cfg(not(feature = "private"))] use uuid::Uuid; +#[cfg(not(feature = "private"))] use windmill_common::s3_helpers::StorageResourceType; -#[cfg(feature = "parquet")] +#[cfg(all(feature = "parquet", not(feature = "private")))] use crate::db::{ApiAuthed, DB}; -#[cfg(feature = "parquet")] +#[cfg(all(feature = "parquet", not(feature = "private")))] use object_store::{ObjectStore, PutMultipartOpts}; -#[cfg(feature = "parquet")] +#[cfg(all(feature = "parquet", not(feature = "private")))] use std::sync::Arc; +#[cfg(not(feature = "private"))] use windmill_common::error; -#[cfg(feature = "parquet")] +#[cfg(all(feature = "parquet", not(feature = "private")))] use windmill_common::{db::UserDB, s3_helpers::ObjectStoreResource}; -#[cfg(feature = "parquet")] +#[cfg(all(feature = "parquet", not(feature = "private")))] use bytes::Bytes; -#[cfg(feature = "parquet")] +#[cfg(all(feature = "parquet", not(feature = "private")))] use futures::Stream; -#[cfg(feature = "parquet")] +#[cfg(all(feature = "parquet", not(feature = "private")))] use axum::response::Response; -#[cfg(feature = "parquet")] +#[cfg(all(feature = "parquet", not(feature = "private")))] use serde::Deserialize; #[derive(Serialize)] +#[cfg(not(feature = "private"))] pub struct UploadFileResponse { pub file_key: String, } #[derive(Deserialize)] +#[cfg(not(feature = "private"))] pub struct LoadImagePreviewQuery { #[allow(dead_code)] pub file_key: String, @@ -37,6 +48,7 @@ pub struct LoadImagePreviewQuery { } #[derive(Deserialize)] +#[cfg(not(feature = "private"))] pub struct DownloadFileQuery { #[allow(dead_code)] pub file_key: String, @@ -46,11 +58,12 @@ pub struct DownloadFileQuery { pub s3_resource_path: Option, } +#[cfg(not(feature = "private"))] pub fn workspaced_service() -> Router { Router::new() } -#[cfg(feature = "parquet")] +#[cfg(all(feature = "parquet", not(feature = "private")))] pub async fn get_workspace_s3_resource<'c>( _authed: &ApiAuthed, _db: &DB, @@ -63,10 +76,12 @@ pub async fn get_workspace_s3_resource<'c>( Ok((None, None)) } +#[cfg(not(feature = "private"))] pub fn get_random_file_name(_file_extension: Option) -> String { unimplemented!("Not implemented in Windmill's Open Source repository") } +#[cfg(not(feature = "private"))] pub async fn get_s3_resource<'c>( _authed: &ApiAuthed, _db: &DB, @@ -82,7 +97,7 @@ pub async fn get_s3_resource<'c>( )) } -#[cfg(feature = "parquet")] +#[cfg(all(feature = "parquet", not(feature = "private")))] pub async fn upload_file_from_req( _s3_client: Arc, _file_key: &str, @@ -94,7 +109,7 @@ pub async fn upload_file_from_req( )) } -#[cfg(feature = "parquet")] +#[cfg(all(feature = "parquet", not(feature = "private")))] pub async fn upload_file_internal( _s3_client: Arc, _file_key: &str, @@ -106,7 +121,7 @@ pub async fn upload_file_internal( )) } -#[cfg(feature = "parquet")] +#[cfg(all(feature = "parquet", not(feature = "private")))] pub async fn download_s3_file_internal( _authed: ApiAuthed, _db: &DB, diff --git a/backend/windmill-api/src/jobs.rs b/backend/windmill-api/src/jobs.rs index cf9e7d06de..8b0b5e5e76 100644 --- a/backend/windmill-api/src/jobs.rs +++ b/backend/windmill-api/src/jobs.rs @@ -26,6 +26,7 @@ use tokio::io::AsyncReadExt; #[cfg(feature = "prometheus")] use tokio::time::Instant; use tower::ServiceBuilder; +use windmill_common::auth::is_super_admin_email; use windmill_common::error::JsonResult; use windmill_common::flow_status::{JobResult, RestartedFrom}; use windmill_common::jobs::{format_completed_job_result, format_result, ENTRYPOINT_OVERRIDE}; @@ -64,7 +65,7 @@ use sqlx::types::JsonRawValue; use sqlx::{types::Uuid, FromRow, Postgres, Transaction}; use tower_http::cors::{Any, CorsLayer}; use urlencoding::encode; -use windmill_audit::audit_ee::{audit_log, AuditAuthor}; +use windmill_audit::audit_oss::{audit_log, AuditAuthor}; use windmill_audit::ActionKind; use windmill_common::worker::{to_raw_value, CUSTOM_TAGS_PER_WORKSPACE}; use windmill_common::{ @@ -3140,28 +3141,27 @@ pub fn add_raw_string( } async fn check_tag_available_for_workspace( + db: &DB, w_id: &str, tag: &Option, authed: &ApiAuthed, ) -> error::Result<()> { if let Some(tag) = tag { - if tag == "" { + if tag.is_empty() { return Ok(()); } let tags = get_scope_tags(authed); + let mut is_tag_available_in_workspace = None; + let mut is_tag_in_workspace_custom_tags = false; - if let Some(tags) = tags { - if !tags.contains(&tag.as_str()) { - return Err(Error::BadRequest(format!( - "Tag {tag} is not available in your scope" - ))); - } + if let Some(tags) = tags.as_ref() { + is_tag_available_in_workspace = Some(tags.contains(&tag.as_str())); } let custom_tags_per_w = CUSTOM_TAGS_PER_WORKSPACE.read().await; if custom_tags_per_w.0.contains(&tag.to_string()) { - Ok(()) + is_tag_in_workspace_custom_tags = true; } else if custom_tags_per_w.1.contains_key(tag) && custom_tags_per_w .1 @@ -3169,21 +3169,38 @@ async fn check_tag_available_for_workspace( .unwrap() .contains(&w_id.to_string()) { - Ok(()) - } else { + is_tag_in_workspace_custom_tags = true; + } + + match is_tag_available_in_workspace { + Some(true) | None => { + if is_tag_in_workspace_custom_tags { + return Ok(()); + } + } + _ => {} + } + + if !is_super_admin_email(db, &authed.email).await? { + if tags.is_some() && is_tag_available_in_workspace.is_some() { + return Err(Error::BadRequest(format!( + "Tag {tag} is not available in your scope" + ))); + } + return Err(error::Error::BadRequest(format!( - "Tag {tag} cannot be used on workspace {w_id}: (CUSTOM_TAGS: {:?})", + "Only super admins are allowed to use tags that are not included in the allowed CUSTOM_TAGS: {:?}", custom_tags_per_w ))); } - } else { - Ok(()) } + + return Ok(()); } #[cfg(feature = "enterprise")] pub async fn check_license_key_valid() -> error::Result<()> { - use windmill_common::ee::LICENSE_KEY_VALID; + use windmill_common::ee_oss::LICENSE_KEY_VALID; let valid = *LICENSE_KEY_VALID.read().await; if !valid { @@ -3509,7 +3526,7 @@ pub async fn run_flow_by_path_inner( let tag = run_query.tag.clone().or(tag); - check_tag_available_for_workspace(&w_id, &tag, &authed).await?; + check_tag_available_for_workspace(&db, &w_id, &tag, &authed).await?; let scheduled_for = run_query.get_scheduled_for(&db).await?; let (email, permissioned_as, push_authed, tx) = @@ -3701,7 +3718,7 @@ pub async fn run_script_by_path_inner( let scheduled_for = run_query.get_scheduled_for(&db).await?; let tag = run_query.tag.clone().or(tag); - check_tag_available_for_workspace(&w_id, &tag, &authed).await?; + check_tag_available_for_workspace(&db, &w_id, &tag, &authed).await?; let (email, permissioned_as, push_authed, tx) = if let Some(on_behalf_of) = on_behalf_of.as_ref() { @@ -3772,7 +3789,7 @@ pub async fn run_workflow_as_code( #[cfg(feature = "enterprise")] check_license_key_valid().await?; - check_tag_available_for_workspace(&w_id, &run_query.tag, &authed).await?; + check_tag_available_for_workspace(&db, &w_id, &run_query.tag, &authed).await?; if *CLOUD_HOSTED { tracing::info!("workflow_as_code_tracing id {i} "); @@ -4369,7 +4386,7 @@ pub async fn run_wait_result_job_by_path_get( drop(tx); let tag = run_query.tag.clone().or(tag); - check_tag_available_for_workspace(&w_id, &tag, &authed).await?; + check_tag_available_for_workspace(&db, &w_id, &tag, &authed).await?; let (email, permissioned_as, push_authed, tx) = if let Some(on_behalf_of) = on_behalf_authed.as_ref() { @@ -4509,7 +4526,7 @@ pub async fn run_wait_result_script_by_path_internal( drop(tx); let tag = run_query.tag.clone().or(tag); - check_tag_available_for_workspace(&w_id, &tag, &authed).await?; + check_tag_available_for_workspace(&db, &w_id, &tag, &authed).await?; let (email, permissioned_as, push_authed, tx) = if let Some(on_behalf_of) = on_behalf_of.as_ref() { @@ -4610,7 +4627,7 @@ pub async fn run_wait_result_script_by_hash( check_scopes(&authed, || format!("run:script/{path}"))?; let tag = run_query.tag.clone().or(tag); - check_tag_available_for_workspace(&w_id, &tag, &authed).await?; + check_tag_available_for_workspace(&db, &w_id, &tag, &authed).await?; let (email, permissioned_as, push_authed, tx) = if let Some(email) = on_behalf_of_email.as_ref() { @@ -4730,7 +4747,7 @@ pub async fn run_wait_result_flow_by_path_internal( drop(tx); let tag = run_query.tag.clone().or(tag); - check_tag_available_for_workspace(&w_id, &tag, &authed).await?; + check_tag_available_for_workspace(&db, &w_id, &tag, &authed).await?; let (email, permissioned_as, push_authed, tx) = if let Some(on_behalf_of_email) = on_behalf_of_email.as_ref() { @@ -4804,7 +4821,7 @@ async fn run_preview_script( } let scheduled_for = run_query.get_scheduled_for(&db).await?; let tag = run_query.tag.clone().or(preview.tag.clone()); - check_tag_available_for_workspace(&w_id, &tag, &authed).await?; + check_tag_available_for_workspace(&db, &w_id, &tag, &authed).await?; let tx = PushIsolationLevel::Isolated(user_db.clone(), authed.clone().into()); let (uuid, tx) = push( @@ -4886,7 +4903,7 @@ async fn run_bundle_preview_script( let scheduled_for = run_query.get_scheduled_for(&db).await?; let tag = run_query.tag.clone().or(preview.tag.clone()); - check_tag_available_for_workspace(&w_id, &tag, &authed).await?; + check_tag_available_for_workspace(&db, &w_id, &tag, &authed).await?; let ltx = PushIsolationLevel::Isolated(user_db.clone(), authed.clone().into()); let args = preview.args.unwrap_or_default(); @@ -5470,7 +5487,7 @@ async fn run_preview_flow_job( } let scheduled_for = run_query.get_scheduled_for(&db).await?; let tag = run_query.tag.clone().or(raw_flow.tag.clone()); - check_tag_available_for_workspace(&w_id, &tag, &authed).await?; + check_tag_available_for_workspace(&db, &w_id, &tag, &authed).await?; let tx = PushIsolationLevel::Isolated(user_db.clone(), authed.clone().into()); let (uuid, tx) = push( @@ -5565,7 +5582,7 @@ pub async fn run_job_by_hash_inner( let scheduled_for = run_query.get_scheduled_for(&db).await?; let tag = run_query.tag.clone().or(tag); - check_tag_available_for_workspace(&w_id, &tag, &authed).await?; + check_tag_available_for_workspace(&db, &w_id, &tag, &authed).await?; let (email, permissioned_as, push_authed, tx) = if let Some(email) = on_behalf_of_email.as_ref() { diff --git a/backend/windmill-api/src/kafka_triggers_ee.rs b/backend/windmill-api/src/kafka_triggers_oss.rs similarity index 76% rename from backend/windmill-api/src/kafka_triggers_ee.rs rename to backend/windmill-api/src/kafka_triggers_oss.rs index 0a24151ae3..9d698e2fbf 100644 --- a/backend/windmill-api/src/kafka_triggers_ee.rs +++ b/backend/windmill-api/src/kafka_triggers_oss.rs @@ -1,14 +1,24 @@ +#[cfg(feature = "private")] +#[allow(unused)] +pub use crate::kafka_triggers_ee::*; + +#[cfg(not(feature = "private"))] use crate::db::DB; +#[cfg(not(feature = "private"))] use axum::Router; +#[cfg(not(feature = "private"))] use serde::{Deserialize, Serialize}; #[derive(Serialize, Deserialize)] +#[cfg(not(feature = "private"))] pub struct KafkaResourceSecurity {} +#[cfg(not(feature = "private"))] pub fn workspaced_service() -> Router { Router::new() } +#[cfg(not(feature = "private"))] pub fn start_kafka_consumers( _db: DB, mut _killpill_rx: tokio::sync::broadcast::Receiver<()>, @@ -17,9 +27,11 @@ pub fn start_kafka_consumers( } #[derive(Serialize, Deserialize)] +#[cfg(not(feature = "private"))] pub enum KafkaTriggerConfigConnection {} #[derive(Serialize, Clone)] +#[cfg(not(feature = "private"))] pub struct KafkaTrigger { pub workspace_id: String, pub path: String, @@ -39,4 +51,4 @@ pub struct KafkaTrigger { #[serde(skip_serializing_if = "Option::is_none")] pub error: Option, pub enabled: bool, -} \ No newline at end of file +} diff --git a/backend/windmill-api/src/lib.rs b/backend/windmill-api/src/lib.rs index d79639f0ab..a415803339 100644 --- a/backend/windmill-api/src/lib.rs +++ b/backend/windmill-api/src/lib.rs @@ -8,27 +8,29 @@ use crate::db::ApiAuthed; #[cfg(feature = "enterprise")] -use crate::ee::ExternalJwks; +use crate::ee_oss::ExternalJwks; #[cfg(feature = "embedding")] use crate::embeddings::load_embeddings_db; #[cfg(feature = "oauth2")] -use crate::oauth2_ee::AllClients; +use crate::oauth2_oss::AllClients; #[cfg(feature = "oauth2")] -use crate::oauth2_ee::SlackVerifier; +use crate::oauth2_oss::SlackVerifier; #[cfg(feature = "smtp")] -use crate::smtp_server_ee::SmtpServer; +use crate::smtp_server_oss::SmtpServer; #[cfg(feature = "mcp")] -use crate::mcp::{setup_mcp_server, Runner as McpRunner}; +use crate::mcp::{extract_and_store_workspace_id, setup_mcp_server, shutdown_mcp_server}; +#[cfg(feature = "mcp")] +use rmcp::transport::streamable_http_server::session::local::LocalSessionManager; + use crate::tracing_init::MyOnFailure; use crate::{ tracing_init::{MyMakeSpan, MyOnResponse}, users::OptAuthed, webhook_util::WebhookShared, }; - #[cfg(feature = "agent_worker_server")] -use agent_workers_ee::AgentCache; +use agent_workers_oss::AgentCache; use anyhow::Context; use argon2::Argon2; @@ -58,22 +60,26 @@ use windmill_common::db::UserDB; use windmill_common::worker::CLOUD_HOSTED; use windmill_common::{utils::GIT_VERSION, BASE_URL, INSTANCE_NAME}; -use crate::scim_ee::has_scim_token; +use crate::scim_oss::has_scim_token; use windmill_common::error::AppError; +#[cfg(all(feature = "agent_worker_server", feature = "private"))] +pub mod agent_workers_ee; #[cfg(feature = "agent_worker_server")] -mod agent_workers_ee; +mod agent_workers_oss; mod ai; mod apps; pub mod args; mod audit; -mod auth; +pub mod auth; mod capture; mod concurrency_groups; mod configs; mod db; mod drafts; +#[cfg(feature = "private")] pub mod ee; +pub mod ee_oss; pub mod embeddings; mod favorite; mod flows; @@ -86,56 +92,95 @@ mod http_trigger_args; mod http_trigger_auth; #[cfg(feature = "http_trigger")] pub mod http_triggers; -mod indexer_ee; +#[cfg(feature = "private")] +pub mod indexer_ee; +mod indexer_oss; +#[cfg(feature = "private")] +mod inkeep_ee; +mod inkeep_oss; mod inputs; mod integration; #[cfg(feature = "postgres_trigger")] mod postgres_triggers; +pub mod openapi; + mod approvals; +#[cfg(all(feature = "enterprise", feature = "private"))] +pub mod apps_ee; #[cfg(feature = "enterprise")] -mod apps_ee; +mod apps_oss; +#[cfg(all(feature = "enterprise", feature = "gcp_trigger", feature = "private"))] +pub mod gcp_triggers_ee; #[cfg(all(feature = "enterprise", feature = "gcp_trigger"))] -mod gcp_triggers_ee; +mod gcp_triggers_oss; +#[cfg(all(feature = "enterprise", feature = "private"))] +pub mod git_sync_ee; #[cfg(feature = "enterprise")] -mod git_sync_ee; +mod git_sync_oss; +#[cfg(all(feature = "parquet", feature = "private"))] +pub mod job_helpers_ee; #[cfg(feature = "parquet")] -mod job_helpers_ee; +mod job_helpers_oss; pub mod job_metrics; pub mod jobs; +#[cfg(all(feature = "enterprise", feature = "kafka", feature = "private"))] +pub mod kafka_triggers_ee; #[cfg(all(feature = "enterprise", feature = "kafka"))] -mod kafka_triggers_ee; +mod kafka_triggers_oss; #[cfg(feature = "mqtt_trigger")] mod mqtt_triggers; +#[cfg(all(feature = "enterprise", feature = "nats", feature = "private"))] +pub mod nats_triggers_ee; #[cfg(all(feature = "enterprise", feature = "nats"))] -mod nats_triggers_ee; -#[cfg(feature = "oauth2")] +mod nats_triggers_oss; +#[cfg(all(feature = "oauth2", feature = "private"))] pub mod oauth2_ee; -mod oidc_ee; +#[cfg(feature = "oauth2")] +pub mod oauth2_oss; +#[cfg(feature = "private")] +pub mod oidc_ee; +mod oidc_oss; mod raw_apps; mod resources; -mod saml_ee; +#[cfg(feature = "private")] +pub mod saml_ee; +mod saml_oss; mod schedule; -mod scim_ee; +#[cfg(feature = "private")] +pub mod scim_ee; +mod scim_oss; mod scripts; mod service_logs; mod settings; mod slack_approvals; +#[cfg(all(feature = "smtp", feature = "private"))] +pub mod smtp_server_ee; #[cfg(feature = "smtp")] -mod smtp_server_ee; +mod smtp_server_oss; +#[cfg(all(feature = "enterprise", feature = "sqs_trigger", feature = "private"))] +pub mod sqs_triggers_ee; #[cfg(all(feature = "enterprise", feature = "sqs_trigger"))] -mod sqs_triggers_ee; -mod teams_approvals_ee; +mod sqs_triggers_oss; +#[cfg(feature = "private")] +pub mod teams_approvals_ee; +mod teams_approvals_oss; mod trigger_helpers; mod static_assets; +#[cfg(all(feature = "stripe", feature = "enterprise", feature = "private"))] +pub mod stripe_ee; #[cfg(all(feature = "stripe", feature = "enterprise"))] -mod stripe_ee; -mod teams_ee; +mod stripe_oss; +#[cfg(feature = "private")] +pub mod teams_ee; +mod teams_oss; mod tracing_init; mod triggers; mod users; -mod users_ee; +#[cfg(feature = "private")] +pub mod users_ee; +mod users_oss; mod utils; mod variables; pub mod webhook_util; @@ -143,9 +188,11 @@ pub mod webhook_util; mod websocket_triggers; mod workers; mod workspaces; -mod workspaces_ee; +#[cfg(feature = "private")] +pub mod workspaces_ee; mod workspaces_export; mod workspaces_extra; +mod workspaces_oss; #[cfg(feature = "mcp")] mod mcp; @@ -218,9 +265,9 @@ type IndexReader = (); type ServiceLogIndexReader = (); #[cfg(feature = "tantivy")] -type IndexReader = windmill_indexer::completed_runs_ee::IndexReader; +type IndexReader = windmill_indexer::completed_runs_oss::IndexReader; #[cfg(feature = "tantivy")] -type ServiceLogIndexReader = windmill_indexer::service_logs_ee::ServiceLogIndexReader; +type ServiceLogIndexReader = windmill_indexer::service_logs_oss::ServiceLogIndexReader; pub async fn run_server( db: DB, @@ -278,7 +325,7 @@ pub async fn run_server( .allow_headers([http::header::CONTENT_TYPE, http::header::AUTHORIZATION]) .allow_origin(Any); - let sp_extension = Arc::new(saml_ee::build_sp_extension().await?); + let sp_extension = Arc::new(saml_oss::build_sp_extension().await?); if server_mode { #[cfg(feature = "embedding")] @@ -317,7 +364,7 @@ pub async fn run_server( let job_helpers_service = { #[cfg(feature = "parquet")] { - job_helpers_ee::workspaced_service() + job_helpers_oss::workspaced_service() } #[cfg(not(feature = "parquet"))] @@ -329,7 +376,7 @@ pub async fn run_server( let kafka_triggers_service = { #[cfg(all(feature = "enterprise", feature = "kafka"))] { - kafka_triggers_ee::workspaced_service() + kafka_triggers_oss::workspaced_service() } #[cfg(not(all(feature = "enterprise", feature = "kafka")))] @@ -341,7 +388,7 @@ pub async fn run_server( let nats_triggers_service = { #[cfg(all(feature = "enterprise", feature = "nats"))] { - nats_triggers_ee::workspaced_service() + nats_triggers_oss::workspaced_service() } #[cfg(not(all(feature = "enterprise", feature = "nats")))] @@ -365,7 +412,7 @@ pub async fn run_server( let gcp_triggers_service = { #[cfg(all(feature = "enterprise", feature = "gcp_trigger"))] { - gcp_triggers_ee::workspaced_service() + gcp_triggers_oss::workspaced_service() } #[cfg(not(all(feature = "enterprise", feature = "gcp_trigger")))] @@ -377,7 +424,7 @@ pub async fn run_server( let sqs_triggers_service = { #[cfg(all(feature = "enterprise", feature = "sqs_trigger"))] { - sqs_triggers_ee::workspaced_service() + sqs_triggers_oss::workspaced_service() } #[cfg(not(all(feature = "enterprise", feature = "sqs_trigger")))] @@ -432,13 +479,13 @@ pub async fn run_server( #[cfg(all(feature = "enterprise", feature = "kafka"))] { let kafka_killpill_rx = killpill_rx.resubscribe(); - kafka_triggers_ee::start_kafka_consumers(db.clone(), kafka_killpill_rx); + kafka_triggers_oss::start_kafka_consumers(db.clone(), kafka_killpill_rx); } #[cfg(all(feature = "enterprise", feature = "nats"))] { let nats_killpill_rx = killpill_rx.resubscribe(); - nats_triggers_ee::start_nats_consumers(db.clone(), nats_killpill_rx); + nats_triggers_oss::start_nats_consumers(db.clone(), nats_killpill_rx); } #[cfg(feature = "postgres_trigger")] @@ -456,13 +503,13 @@ pub async fn run_server( #[cfg(all(feature = "enterprise", feature = "sqs_trigger"))] { let sqs_killpill_rx = killpill_rx.resubscribe(); - sqs_triggers_ee::start_sqs(db.clone(), sqs_killpill_rx); + sqs_triggers_oss::start_sqs(db.clone(), sqs_killpill_rx); } #[cfg(all(feature = "enterprise", feature = "gcp_trigger"))] { let gcp_killpill_rx = killpill_rx.resubscribe(); - gcp_triggers_ee::start_consuming_gcp_pubsub_event(db.clone(), gcp_killpill_rx); + gcp_triggers_oss::start_consuming_gcp_pubsub_event(db.clone(), gcp_killpill_rx); } } @@ -477,27 +524,24 @@ pub async fn run_server( // Setup MCP server #[allow(unused_variables)] - let (mcp_router, mcp_main_ct, mcp_service_ct) = { + let (mcp_router, mcp_session_manager) = { #[cfg(feature = "mcp")] if server_mode || mcp_mode { - let (mcp_sse_server, mcp_router) = setup_mcp_server(addr, "/api/mcp/w/:workspace_id")?; - #[cfg(feature = "mcp")] - let mcp_main_ct = mcp_sse_server.config.ct.clone(); // Token to signal shutdown *to* MCP - #[cfg(feature = "mcp")] - let mcp_service_ct = mcp_sse_server.with_service(McpRunner::new); // Token to wait for MCP *service* shutdown - (mcp_router, Some(mcp_main_ct), Some(mcp_service_ct)) + let (mcp_router, mcp_session_manager) = setup_mcp_server().await?; + let mcp_middleware = axum::middleware::from_fn(extract_and_store_workspace_id); + (mcp_router.layer(mcp_middleware), Some(mcp_session_manager)) } else { - (Router::new(), None, None) + (Router::new(), Option::>::None) } #[cfg(not(feature = "mcp"))] - (Router::new(), None::<()>, None::<()>) + (Router::new(), Option::<()>::None) }; #[cfg(feature = "agent_worker_server")] let (agent_workers_router, agent_workers_bg_processor, agent_workers_killpill_tx) = if server_mode { - agent_workers_ee::workspaced_service(db.clone(), _base_internal_url.clone()) + agent_workers_oss::workspaced_service(db.clone(), _base_internal_url.clone()) } else { (Router::new(), vec![], None) }; @@ -535,7 +579,7 @@ pub async fn run_server( .nest("/oauth", { #[cfg(feature = "oauth2")] { - oauth2_ee::workspaced_service() + oauth2_oss::workspaced_service() } #[cfg(not(feature = "oauth2"))] @@ -552,7 +596,8 @@ pub async fn run_server( ) .nest("/variables", variables::workspaced_service()) .nest("/workspaces", workspaces::workspaced_service()) - .nest("/oidc", oidc_ee::workspaced_service()) + .nest("/oidc", oidc_oss::workspaced_service()) + .nest("/openapi", openapi::openapi_service()) .nest("/http_triggers", http_triggers_service) .nest("/websocket_triggers", websocket_triggers_service) .nest("/kafka_triggers", kafka_triggers_service) @@ -579,22 +624,23 @@ pub async fn run_server( .nest("/schedules", schedule::global_service()) .nest("/embeddings", embeddings::global_service()) .nest("/ai", ai::global_service()) + .nest("/inkeep", inkeep_oss::global_service()) .route_layer(from_extractor::()) .route_layer(from_extractor::()) .nest("/jobs", jobs::global_root_service()) .nest( "/srch/w/:workspace_id/index", - indexer_ee::workspaced_service(), + indexer_oss::workspaced_service(), ) - .nest("/srch/index", indexer_ee::global_service()) - .nest("/oidc", oidc_ee::global_service()) + .nest("/srch/index", indexer_oss::global_service()) + .nest("/oidc", oidc_oss::global_service()) .nest( "/saml", - saml_ee::global_service().layer(Extension(Arc::clone(&sp_extension))), + saml_oss::global_service().layer(Extension(Arc::clone(&sp_extension))), ) .nest( "/scim", - scim_ee::global_service() + scim_oss::global_service() .route_layer(axum::middleware::from_fn(has_scim_token)), ) .nest("/concurrency_groups", concurrency_groups::global_service()) @@ -602,7 +648,7 @@ pub async fn run_server( .nest("/apps_u", { #[cfg(feature = "enterprise")] { - apps_ee::global_unauthed_service() + apps_oss::global_unauthed_service() } #[cfg(not(feature = "enterprise"))] @@ -616,12 +662,12 @@ pub async fn run_server( .layer(from_extractor::()) .layer(cors.clone()), ) - .nest("/mcp/w/:workspace_id", mcp_router) + .nest("/mcp/w/:workspace_id/sse", mcp_router) .layer(from_extractor::()) .nest("/agent_workers", { #[cfg(feature = "agent_worker_server")] { - agent_workers_ee::global_service().layer(Extension(agent_cache.clone())) + agent_workers_oss::global_service().layer(Extension(agent_cache.clone())) } #[cfg(not(feature = "agent_worker_server"))] { @@ -646,7 +692,7 @@ pub async fn run_server( .nest("/teams", { #[cfg(feature = "enterprise")] { - teams_ee::teams_service() + teams_oss::teams_service() } #[cfg(not(feature = "enterprise"))] @@ -660,12 +706,12 @@ pub async fn run_server( ) .route( "/w/:workspace_id/jobs/teams_approval/:job_id", - get(teams_approvals_ee::request_teams_approval), + get(teams_approvals_oss::request_teams_approval), ) .nest("/w/:workspace_id/github_app", { #[cfg(feature = "enterprise")] { - git_sync_ee::workspaced_service() + git_sync_oss::workspaced_service() } #[cfg(not(feature = "enterprise"))] @@ -674,7 +720,7 @@ pub async fn run_server( .nest("/github_app", { #[cfg(feature = "enterprise")] { - git_sync_ee::global_service() + git_sync_oss::global_service() } #[cfg(not(feature = "enterprise"))] @@ -695,7 +741,7 @@ pub async fn run_server( .nest("/oauth", { #[cfg(feature = "oauth2")] { - oauth2_ee::global_service().layer(Extension(Arc::clone(&sp_extension))) + oauth2_oss::global_service().layer(Extension(Arc::clone(&sp_extension))) } #[cfg(not(feature = "oauth2"))] @@ -721,7 +767,7 @@ pub async fn run_server( { #[cfg(all(feature = "enterprise", feature = "gcp_trigger"))] { - gcp_triggers_ee::gcp_push_route_handler() + gcp_triggers_oss::gcp_push_route_handler() } #[cfg(not(all(feature = "enterprise", feature = "gcp_trigger")))] { @@ -775,16 +821,9 @@ pub async fn run_server( tracing::info!("Graceful shutdown of server"); #[cfg(feature = "mcp")] - { - if let Some(mcp_main_ct) = mcp_main_ct { - tracing::info!("Received shutdown signal, cancelling MCP server..."); - mcp_main_ct.cancel(); - } - if let Some(mcp_service_ct) = mcp_service_ct { - tracing::info!("Waiting for MCP service cancellation..."); - mcp_service_ct.cancelled().await; - tracing::info!("MCP service cancelled."); - } + if let Some(mcp_session_manager) = mcp_session_manager { + shutdown_mcp_server(mcp_session_manager).await; + tracing::info!("MCP server shutdown"); } }); @@ -842,7 +881,7 @@ async fn ee_license() -> &'static str { #[cfg(feature = "enterprise")] async fn ee_license() -> String { - use windmill_common::ee::{LICENSE_KEY_ID, LICENSE_KEY_VALID}; + use windmill_common::ee_oss::{LICENSE_KEY_ID, LICENSE_KEY_VALID}; if *LICENSE_KEY_VALID.read().await { LICENSE_KEY_ID.read().await.clone() diff --git a/backend/windmill-api/src/mcp.rs b/backend/windmill-api/src/mcp.rs index 431e121ba8..be7d5ea2f7 100644 --- a/backend/windmill-api/src/mcp.rs +++ b/backend/windmill-api/src/mcp.rs @@ -1,11 +1,10 @@ use std::borrow::Cow; use std::collections::HashMap; -use std::net::SocketAddr; use std::sync::Arc; use axum::body::to_bytes; use axum::Router; -use rmcp::transport::sse_server::{SseServer, SseServerConfig}; +use axum::{extract::Path, http::Request, middleware::Next, response::Response}; use rmcp::{ handler::server::ServerHandler, model::*, @@ -17,7 +16,6 @@ use serde_json::Value; use sql_builder::prelude::*; use sqlx::FromRow; use tokio::try_join; -use tokio_util::sync::CancellationToken; use windmill_common::db::UserDB; use windmill_common::worker::to_raw_value; use windmill_common::{DB, HUB_BASE_URL}; @@ -29,6 +27,9 @@ use crate::jobs::{ run_wait_result_flow_by_path_internal, run_wait_result_script_by_path_internal, RunJobQuery, }; use crate::HTTP_CLIENT; +use rmcp::transport::streamable_http_server::{ + session::local::LocalSessionManager, SessionManager, StreamableHttpService, +}; use windmill_common::utils::{query_elems_from_hub, StripPath}; /// Transforms the path for workspace scripts/flows. @@ -856,28 +857,44 @@ impl ServerHandler for Runner { }) }; - let authed = context - .req_extensions - .get::() - .ok_or_else(|| Error::internal_error("ApiAuthed not found", None))?; - let db = context - .req_extensions - .get::() - .ok_or_else(|| Error::internal_error("DB not found", None))?; - let user_db = context - .req_extensions - .get::() - .ok_or_else(|| Error::internal_error("UserDB not found", None))?; + let http_parts = context + .extensions + .get::() + .ok_or_else(|| { + tracing::error!("http::request::Parts not found"); + Error::internal_error("http::request::Parts not found", None) + })?; + + let authed = http_parts.extensions.get::().ok_or_else(|| { + tracing::error!("ApiAuthed Axum extension not found"); + Error::internal_error("ApiAuthed Axum extension not found", None) + })?; + let db = http_parts.extensions.get::().ok_or_else(|| { + tracing::error!("DB Axum extension not found"); + Error::internal_error("DB Axum extension not found", None) + })?; + let user_db = http_parts.extensions.get::().ok_or_else(|| { + tracing::error!("UserDB Axum extension not found"); + Error::internal_error("UserDB Axum extension not found", None) + })?; let args = parse_args(request.arguments)?; + let workspace_id = http_parts + .extensions + .get::() + .ok_or_else(|| { + tracing::error!("WorkspaceId not found"); + Error::internal_error("WorkspaceId not found", None) + }) + .map(|w_id| w_id.0.clone())?; + let (tool_type, path, is_hub) = Runner::reverse_transform(&request.name).unwrap_or_default(); let item_schema = if is_hub { Runner::get_hub_script_schema(&format!("hub/{}", path), db).await? } else { - Runner::get_item_schema(&path, user_db, authed, &context.workspace_id, &tool_type) - .await? + Runner::get_item_schema(&path, user_db, authed, &workspace_id, &tool_type).await? }; let schema_obj = if let Some(ref s) = item_schema { @@ -906,8 +923,6 @@ impl ServerHandler for Runner { } else { windmill_queue::PushArgsOwned::default() }; - - let w_id = context.workspace_id.clone(); let script_or_flow_path = if is_hub { StripPath(format!("hub/{}", path)) } else { @@ -922,7 +937,7 @@ impl ServerHandler for Runner { script_or_flow_path, authed.clone(), user_db.clone(), - w_id.clone(), + workspace_id.clone(), push_args, ) .await @@ -934,7 +949,7 @@ impl ServerHandler for Runner { authed.clone(), user_db.clone(), push_args, - w_id.clone(), + workspace_id.clone(), ) .await }; @@ -978,19 +993,38 @@ impl ServerHandler for Runner { _request: Option, mut _context: RequestContext, ) -> Result { - let workspace_id = _context.workspace_id.clone(); - let db = _context - .req_extensions - .get::() - .ok_or_else(|| Error::internal_error("DB not found", None))?; - let user_db = _context - .req_extensions - .get::() - .ok_or_else(|| Error::internal_error("UserDB not found", None))?; - let authed = _context - .req_extensions - .get::() - .ok_or_else(|| Error::internal_error("ApiAuthed not found", None))?; + let http_parts = _context + .extensions + .get::() + .ok_or_else(|| { + tracing::error!("http::request::Parts not found"); + Error::internal_error("http::request::Parts not found", None) + })?; + + let db = http_parts.extensions.get::().ok_or_else(|| { + tracing::error!("DB Axum extension not found"); + Error::internal_error("DB Axum extension not found", None) + })?; + + let user_db = http_parts.extensions.get::().ok_or_else(|| { + tracing::error!("UserDB Axum extension not found"); + Error::internal_error("UserDB Axum extension not found", None) + })?; + + let authed = http_parts.extensions.get::().ok_or_else(|| { + tracing::error!("ApiAuthed Axum extension not found"); + Error::internal_error("ApiAuthed Axum extension not found", None) + })?; + + let workspace_id = http_parts + .extensions + .get::() + .ok_or_else(|| { + tracing::error!("WorkspaceId not found"); + Error::internal_error("WorkspaceId not found", None) + }) + .map(|w_id| w_id.0.clone())?; + let owned_scope = authed.scopes.as_ref().and_then(|scopes| { scopes .iter() @@ -1127,15 +1161,54 @@ impl ServerHandler for Runner { } } -pub fn setup_mcp_server(addr: SocketAddr, path: &str) -> anyhow::Result<(SseServer, Router)> { - let config = SseServerConfig { - bind: addr, - sse_path: "/sse".to_string(), - post_path: "/message".to_string(), - full_message_path: path.to_string(), - ct: CancellationToken::new(), - sse_keep_alive: None, +#[derive(Clone, Debug)] +pub struct WorkspaceId(pub String); + +pub async fn extract_and_store_workspace_id( + Path(params): Path, + mut request: Request, + next: Next, +) -> Response { + let workspace_id = params; + request.extensions_mut().insert(WorkspaceId(workspace_id)); + next.run(request).await +} + +pub async fn setup_mcp_server() -> anyhow::Result<(Router, Arc)> { + let session_manager = Arc::new(LocalSessionManager::default()); + let service_config = Default::default(); + let service = StreamableHttpService::new( + || Ok(Runner::new()), + session_manager.clone(), + service_config, + ); + + let router = axum::Router::new().nest_service("/", service); + Ok((router, session_manager)) +} + +pub async fn shutdown_mcp_server(session_manager: Arc) { + let session_ids_to_close = { + let sessions_map = session_manager.sessions.read().await; + sessions_map.keys().cloned().collect::>() }; - Ok(SseServer::new(config)) + if !session_ids_to_close.is_empty() { + tracing::info!( + "Closing {} active MCP session(s)...", + session_ids_to_close.len() + ); + let close_futures = session_ids_to_close + .iter() + .map(|session_id| { + let manager_clone = session_manager.clone(); + async move { + if let Err(_) = manager_clone.close_session(session_id).await { + tracing::warn!("Error closing MCP session"); + } + } + }) + .collect::>(); + futures::future::join_all(close_futures).await; + } } diff --git a/backend/windmill-api/src/mqtt_triggers.rs b/backend/windmill-api/src/mqtt_triggers.rs index 9f3fd19827..7cc3f3c4e0 100644 --- a/backend/windmill-api/src/mqtt_triggers.rs +++ b/backend/windmill-api/src/mqtt_triggers.rs @@ -39,7 +39,7 @@ use sql_builder::{bind::Bind, SqlBuilder}; use sqlx::{FromRow, Type}; use std::collections::HashMap; use std::time::Duration; -use windmill_audit::{audit_ee::audit_log, ActionKind}; +use windmill_audit::{audit_oss::audit_log, ActionKind}; use windmill_common::{ db::UserDB, error::{self, JsonResult}, @@ -478,7 +478,7 @@ pub async fn test_mqtt_connection( test_postgres; let mqtt_resource = try_get_resource_from_db_as::( - authed, + &authed, Some(user_db), &db, &mqtt_resource_path, @@ -1253,7 +1253,7 @@ impl MqttConfig { } } let mqtt_resource = try_get_resource_from_db_as::( - authed, + &authed, Some(UserDB::new(db.clone())), db, mqtt_resource_path, diff --git a/backend/windmill-api/src/nats_triggers_ee.rs b/backend/windmill-api/src/nats_triggers_oss.rs similarity index 79% rename from backend/windmill-api/src/nats_triggers_ee.rs rename to backend/windmill-api/src/nats_triggers_oss.rs index 649d3a3837..28bf1e70e2 100644 --- a/backend/windmill-api/src/nats_triggers_ee.rs +++ b/backend/windmill-api/src/nats_triggers_oss.rs @@ -1,22 +1,34 @@ +#[cfg(feature = "private")] +#[allow(unused)] +pub use crate::nats_triggers_ee::*; + +#[cfg(not(feature = "private"))] use crate::db::DB; +#[cfg(not(feature = "private"))] use axum::Router; +#[cfg(not(feature = "private"))] use serde::{Deserialize, Serialize}; +#[cfg(not(feature = "private"))] #[derive(Serialize, Deserialize)] pub struct NatsResourceAuth {} +#[cfg(not(feature = "private"))] pub fn workspaced_service() -> Router { Router::new() } +#[cfg(not(feature = "private"))] pub fn start_nats_consumers(_db: DB, mut _killpill_rx: tokio::sync::broadcast::Receiver<()>) -> () { // implementation is not open source } #[derive(Serialize, Deserialize)] +#[cfg(not(feature = "private"))] pub enum NatsTriggerConfigConnection {} #[derive(Serialize, Clone)] +#[cfg(not(feature = "private"))] pub struct NatsTrigger { pub workspace_id: String, pub path: String, @@ -40,4 +52,4 @@ pub struct NatsTrigger { #[serde(skip_serializing_if = "Option::is_none")] pub error: Option, pub enabled: bool, -} \ No newline at end of file +} diff --git a/backend/windmill-api/src/oauth2_ee.rs b/backend/windmill-api/src/oauth2_oss.rs similarity index 76% rename from backend/windmill-api/src/oauth2_ee.rs rename to backend/windmill-api/src/oauth2_oss.rs index 49d2155cf3..04e0e1b202 100644 --- a/backend/windmill-api/src/oauth2_ee.rs +++ b/backend/windmill-api/src/oauth2_oss.rs @@ -1,3 +1,7 @@ +#[cfg(feature = "private")] +#[allow(unused)] +pub use crate::oauth2_ee::*; + /* * Author: Ruben Fiszel * Copyright: Windmill Labs, Inc 2022 @@ -6,39 +10,50 @@ * LICENSE-AGPL for a copy of the license. */ +#[cfg(not(feature = "private"))] use std::{collections::HashMap, fmt::Debug}; +#[cfg(not(feature = "private"))] use axum::{routing::get, Json, Router}; +#[cfg(not(feature = "private"))] use hmac::Mac; -#[cfg(feature = "oauth2")] +#[cfg(all(feature = "oauth2", not(feature = "private")))] use itertools::Itertools; -#[cfg(feature = "oauth2")] +#[cfg(all(feature = "oauth2", not(feature = "private")))] use oauth2::{Client as OClient, *}; +#[cfg(not(feature = "private"))] use serde::{Deserialize, Serialize}; +#[cfg(not(feature = "private"))] use sqlx::{Postgres, Transaction}; -#[cfg(feature = "oauth2")] +#[cfg(all(feature = "oauth2", not(feature = "private")))] use windmill_common::more_serde::maybe_number_opt; -#[cfg(feature = "oauth2")] +#[cfg(all(feature = "oauth2", not(feature = "private")))] use crate::OAUTH_CLIENTS; +#[cfg(not(feature = "private"))] use windmill_common::error; +#[cfg(not(feature = "private"))] use windmill_common::oauth2::*; +#[cfg(not(feature = "private"))] use crate::db::DB; +#[cfg(not(feature = "private"))] use std::str; +#[cfg(not(feature = "private"))] pub fn global_service() -> Router { Router::new() .route("/list_logins", get(list_logins)) .route("/list_connects", get(list_connects)) } +#[cfg(not(feature = "private"))] pub fn workspaced_service() -> Router { Router::new() } -#[cfg(feature = "oauth2")] +#[cfg(all(feature = "oauth2", not(feature = "private")))] #[derive(Debug, Clone)] pub struct ClientWithScopes { _client: OClient, @@ -48,9 +63,10 @@ pub struct ClientWithScopes { _allowed_domains: Option>, _userinfo_url: Option, } -#[cfg(feature = "oauth2")] +#[cfg(all(feature = "oauth2", not(feature = "private")))] pub type BasicClientsMap = HashMap; +#[cfg(not(feature = "private"))] #[derive(Clone, Debug, Serialize, Deserialize)] pub struct OAuthConfig { auth_url: String, @@ -62,6 +78,7 @@ pub struct OAuthConfig { req_body_auth: Option, } +#[cfg(not(feature = "private"))] #[derive(Clone, Debug, Serialize, Deserialize)] pub struct OAuthClient { id: String, @@ -71,7 +88,7 @@ pub struct OAuthClient { login_config: Option, } -#[cfg(feature = "oauth2")] +#[cfg(all(feature = "oauth2", not(feature = "private")))] #[derive(Debug)] pub struct AllClients { pub logins: BasicClientsMap, @@ -79,7 +96,7 @@ pub struct AllClients { pub slack: Option, } -#[cfg(feature = "oauth2")] +#[cfg(all(feature = "oauth2", not(feature = "private")))] pub async fn build_oauth_clients( _base_url: &str, _oauths_from_config: Option>, @@ -93,7 +110,7 @@ pub async fn build_oauth_clients( }); } -#[cfg(feature = "oauth2")] +#[cfg(all(feature = "oauth2", not(feature = "private")))] #[derive(Clone, Debug, Deserialize, Serialize)] pub struct TokenResponse { access_token: AccessToken, @@ -107,17 +124,20 @@ pub struct TokenResponse { scope: Option>, } +#[cfg(not(feature = "private"))] #[derive(Serialize)] struct Logins { oauth: Vec, saml: Option, } +#[cfg(not(feature = "private"))] async fn list_logins() -> error::JsonResult { // Implementation is not open source return Ok(Json(Logins { oauth: vec![], saml: None })); } -#[cfg(feature = "oauth2")] +#[allow(unused)] +#[cfg(all(feature = "oauth2", not(feature = "private")))] async fn list_connects() -> error::JsonResult> { Ok(Json( (&OAUTH_CLIENTS.read().await.connects) @@ -127,12 +147,14 @@ async fn list_connects() -> error::JsonResult> { )) } -#[cfg(not(feature = "oauth2"))] -async fn list_connects() -> error::JsonResult> { +#[allow(unused)] +#[cfg(not(all(feature = "oauth2", not(feature = "private"))))] +async fn list_connects() -> windmill_common::error::JsonResult> { // Implementation is not open source - return Ok(Json(vec![])); + return Ok(axum::Json(vec![])); } +#[cfg(not(feature = "private"))] pub async fn _refresh_token<'c>( _tx: Transaction<'c, Postgres>, _path: &str, @@ -146,6 +168,7 @@ pub async fn _refresh_token<'c>( )) } +#[cfg(not(feature = "private"))] pub async fn check_nb_of_user(db: &DB) -> error::Result<()> { let nb_users_sso = sqlx::query_scalar!("SELECT COUNT(*) FROM password WHERE login_type != 'password'",) @@ -171,10 +194,11 @@ pub async fn check_nb_of_user(db: &DB) -> error::Result<()> { } #[derive(Clone, Debug)] +#[cfg(not(feature = "private"))] pub struct SlackVerifier { _mac: HmacSha256, } - +#[cfg(not(feature = "private"))] impl SlackVerifier { pub fn new>(secret: S) -> anyhow::Result { HmacSha256::new_from_slice(secret.as_ref()) diff --git a/backend/windmill-api/src/oidc_ee.rs b/backend/windmill-api/src/oidc_oss.rs similarity index 68% rename from backend/windmill-api/src/oidc_ee.rs rename to backend/windmill-api/src/oidc_oss.rs index 248b990f54..4042c2a53b 100644 --- a/backend/windmill-api/src/oidc_ee.rs +++ b/backend/windmill-api/src/oidc_oss.rs @@ -1,3 +1,7 @@ +#[cfg(feature = "private")] +#[allow(unused)] +pub use crate::oidc_ee::*; + /* * Author: Ruben Fiszel * Copyright: Windmill Labs, Inc 2023 @@ -6,12 +10,15 @@ * LICENSE-AGPL for a copy of the license. */ +#[cfg(not(feature = "private"))] use axum::Router; +#[cfg(not(feature = "private"))] pub fn global_service() -> Router { Router::new() } +#[cfg(not(feature = "private"))] pub fn workspaced_service() -> Router { Router::new() } diff --git a/backend/windmill-api/src/openapi.rs b/backend/windmill-api/src/openapi.rs new file mode 100644 index 0000000000..c0e1d046b6 --- /dev/null +++ b/backend/windmill-api/src/openapi.rs @@ -0,0 +1,984 @@ +use std::{ + collections::{HashMap, HashSet}, + fmt::Display, +}; + +use anyhow::anyhow; +use axum::{ + body::Body, extract::Path, http, response::Response, routing::post, Extension, Json, Router, +}; +use http::{header, HeaderValue, Method, StatusCode}; +use indexmap::IndexMap; +use itertools::Itertools; +use serde::{Deserialize, Serialize}; +use serde_json::{to_value, Map, Value}; +use sqlx::PgConnection; +use url::Url; +use windmill_common::{ + db::UserDB, + error::{Error, Result}, + utils::{deserialize_url, empty_as_none, is_empty, RunnableKind}, + DB, +}; + +use crate::db::ApiAuthed; + +#[cfg(feature = "http_trigger")] +use crate::{ + http_trigger_args::HttpMethod, http_trigger_auth::ApiKeyAuthentication, + http_triggers::AuthenticationMethod, resources::try_get_resource_from_db_as, +}; + +lazy_static::lazy_static! { + static ref DEFAULT_OPENAPI_INFO_OBJECT: Info = Info { + title: "Windmill API".to_string(), + version: "1.0.0".to_string(), + ..Default::default() + }; +} + +const DEFAULT_OPENAPI_GENERATED_VERSION: &'static str = "3.1.0"; +const JWT_SECURITY_SCHEME: &'static str = "JwtAuth"; +const BASIC_HTTP_AUTH_SCHEME: &'static str = "BasicHttp"; + +const DEFAULT_REQUEST_KEY: &'static str = "defaultRequest"; +const DEFAULT_ASYNC_RESPONSE_KEY: &'static str = "AsyncResponse"; +const DEFAULT_SYNC_RESPONSE_KEY: &'static str = "SyncResponse"; +const DEFAULT_PAYLOAD_PARAM_KEY: &'static str = "PayloadParam"; + +pub fn openapi_service() -> Router { + Router::new() + .route("/generate", post(generate_openapi_spec)) + .route("/download", post(download_spec)) +} + +#[derive(Debug, Deserialize, Clone, Copy)] +#[serde(rename_all = "lowercase")] +pub enum Format { + JSON, + YAML, +} + +impl Display for Format { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let format = match self { + Format::JSON => "json", + Format::YAML => "yaml", + }; + write!(f, "{}", format) + } +} + +impl Default for Format { + fn default() -> Self { + Self::YAML + } +} + +#[derive(Debug, Default, Deserialize, Serialize)] +struct Contact { + #[serde(skip_serializing_if = "is_empty")] + name: Option, + #[serde( + default, + deserialize_with = "deserialize_url", + skip_serializing_if = "Option::is_none" + )] + url: Option, + #[serde(skip_serializing_if = "is_empty")] + email: Option, +} + +#[derive(Debug, Default, Deserialize, Serialize)] +struct License { + name: String, + #[serde(skip_serializing_if = "is_empty")] + identifier: Option, + #[serde( + default, + deserialize_with = "deserialize_url", + skip_serializing_if = "Option::is_none" + )] + url: Option, +} + +#[derive(Debug, Default, Deserialize, Serialize)] +pub struct Info { + title: String, + version: String, + #[serde(skip_serializing_if = "is_empty")] + description: Option, + #[serde(skip_serializing_if = "Option::is_none")] + contact: Option, + #[serde(skip_serializing_if = "Option::is_none")] + license: Option, +} + +#[derive(Debug, Serialize, Deserialize)] +struct Server { + url: String, + #[serde(skip_serializing_if = "is_empty")] + description: Option, + #[serde(skip_serializing_if = "Option::is_none")] + variables: Option>, +} + +#[derive(Debug)] +pub enum SecurityScheme { + BearerJwt, + BasicHttp, + ApiKey(String), +} +#[derive(Debug)] +pub struct WebhookConfig { + runnable_kind: RunnableKind, +} + +impl WebhookConfig { + pub fn new(runnable_kind: RunnableKind) -> Self { + Self { runnable_kind } + } +} + +#[derive(Debug)] +pub struct HttpRouteConfig { + method: Method, +} + +impl HttpRouteConfig { + pub fn new(method: Method) -> Self { + Self { method } + } +} + +#[derive(Debug)] +pub enum Kind { + Webhook(WebhookConfig), + HttpRoute(HttpRouteConfig), +} + +#[derive(Debug)] +pub struct FuturePath { + route_path: String, + kind: Kind, + is_async: Option, + summary: Option, + description: Option, + security_scheme: Option, +} + +impl FuturePath { + pub fn new( + route_path: String, + kind: Kind, + is_async: Option, + summary: Option, + description: Option, + security_scheme: Option, + ) -> FuturePath { + FuturePath { route_path, kind, is_async, summary, description, security_scheme } + } +} + +fn from_route_path_to_openapi_path( + route_path: &str, + kind: &Kind, +) -> Result<(Vec, Option)> { + let mut openapi_path = String::new(); + let mut parameters = Vec::new(); + + for segment in route_path.split('/') { + if segment.starts_with(':') { + let param_name = &segment[1..]; + + if param_name.is_empty() { + return Err(anyhow!("Empty parameter name in path: {}", route_path).into()); + } + + openapi_path.push_str(&format!("/{{{}}}", param_name)); + parameters.push(serde_json::json!({ + "name": param_name, + "in": "path", + "required": true, + "schema": { "type": "string" } + })); + } else if !segment.is_empty() { + openapi_path.push('/'); + openapi_path.push_str(segment); + } else { + openapi_path.push('/'); + } + } + + let parameters_json = if parameters.is_empty() { + None + } else { + Some(Value::Array(parameters)) + }; + + let prefix = match kind { + Kind::HttpRoute(_) => "", + Kind::Webhook(WebhookConfig { runnable_kind }) => match runnable_kind { + RunnableKind::Script => "p", + RunnableKind::Flow => "f", + }, + }; + + let normalized_path = if openapi_path.starts_with('/') { + format!("{prefix}{openapi_path}") + } else { + format!("{}/{}", prefix, openapi_path) + }; + + let route_paths = if prefix.is_empty() { + vec![normalized_path] + } else { + vec![ + format!("/run/{}", &normalized_path), + format!("/run_wait_result/{}", &normalized_path), + ] + }; + + Ok((route_paths, parameters_json)) +} + +fn get_servers_component(url: &str, kind: &Kind) -> Server { + let url = url.trim_end_matches('/'); + + let server = match kind { + Kind::HttpRoute(_) => { + Server { url: format!("{}/api/r", url), description: None, variables: None } + } + Kind::Webhook(_) => Server { + url: format!("{}/api/w/{{workspace}}/jobs", url), + variables: Some(HashMap::from([( + "workspace".to_string(), + serde_json::json!({ + "default": "test", + "description": "Workspace identifier" + }), + )])), + description: None, + }, + }; + + server +} + +fn generate_paths( + paths: Vec, + url: Option<&Url>, +) -> Result>> { + let mut map: IndexMap> = IndexMap::new(); + + let generate_default_request = || { + serde_json::json!({ + "$ref": format!("#/components/requestBodies/{DEFAULT_REQUEST_KEY}") + }) + }; + + let generate_response = |is_async: bool| { + let responses = if is_async { + serde_json::json!({ + "200": { + "$ref": format!("#/components/responses/{DEFAULT_ASYNC_RESPONSE_KEY}") + } + }) + } else { + serde_json::json!(serde_json::json!({ + "200": { + "$ref": format!("#/components/responses/{DEFAULT_SYNC_RESPONSE_KEY}") + } + })) + }; + + responses + }; + + let get_security_scheme = |security_scheme: Option<&SecurityScheme>| -> Vec { + if let Some(security_scheme) = security_scheme { + let scheme = match security_scheme { + SecurityScheme::ApiKey(api_key) => header_to_pascal_case(&api_key), + SecurityScheme::BearerJwt => JWT_SECURITY_SCHEME.to_owned(), + SecurityScheme::BasicHttp => BASIC_HTTP_AUTH_SCHEME.to_owned(), + }; + + vec![serde_json::json!({ + scheme: [] + })] + } else { + vec![] + } + }; + + let mut duplicate_webhooks = HashSet::new(); + + for path in paths { + let (route_paths, parameters) = + from_route_path_to_openapi_path(&path.route_path, &path.kind)?; + + for route_path in route_paths { + let path_object = map.entry(route_path.clone()).or_insert_with(|| { + let mut path_object = IndexMap::new(); + + if let Some(url) = url { + let servers = get_servers_component(url.as_str(), &path.kind); + path_object.insert("servers".to_string(), to_value(vec![servers]).unwrap()); + } + + if parameters.is_some() { + path_object.insert( + "parameters".to_string(), + to_value(parameters.clone()).unwrap(), + ); + } + + path_object + }); + + let is_async; + + let (methods, is_webhook) = match &path.kind { + Kind::Webhook(_) => { + if !duplicate_webhooks.insert(route_path.clone()) { + return Err(anyhow!("Found duplicate webhook: {}", path.route_path).into()); + } + is_async = route_path.starts_with("/run/"); + let methods = if is_async { + vec![Method::POST] + } else { + vec![Method::GET, Method::POST] + }; + + (methods, true) + } + Kind::HttpRoute(HttpRouteConfig { method }) => { + if path_object.get(&method.to_string()).is_some() { + return Err(anyhow!("Found duplicate route: {}", path.route_path).into()); + } + is_async = path.is_async.unwrap_or(true); + (vec![method.to_owned()], false) + } + }; + + for method in methods { + let mut method_map = IndexMap::new(); + + if let Some(summary) = path.summary.as_ref().filter(|s| !s.is_empty()) { + method_map.insert("summary", Value::String(summary.to_owned())); + } + + if let Some(description) = path.description.as_ref().filter(|s| !s.is_empty()) { + method_map.insert("description", Value::String(description.to_owned())); + } + + method_map.insert( + "security", + to_value(get_security_scheme(path.security_scheme.as_ref()))?, + ); + + if method != Method::GET { + method_map.insert("requestBody", generate_default_request()); + } else if is_webhook { + method_map.insert( + "parameters", + Value::Array(vec![serde_json::json!({ + "$ref": format!("#/components/parameters/{DEFAULT_PAYLOAD_PARAM_KEY}") + })]), + ); + } + + method_map.insert("responses", generate_response(is_async)); + + path_object.insert(method.to_string().to_lowercase(), to_value(&method_map)?); + } + } + } + + return Ok(map); +} + +pub fn transform_to_minified_postgres_regex(glob: &str) -> String { + let mut regex = String::from("^"); + for ch in glob.chars() { + match ch { + '*' => regex.push_str(".*"), + '.' | '+' | '(' | ')' | '|' | '^' | '$' | '{' | '}' | '[' | ']' | '\\' => { + regex.push('\\'); + regex.push(ch); + } + _ => regex.push(ch), + } + } + + regex.push('$'); + regex +} + +#[derive(Debug, Default)] +pub struct ServerToSet { + pub http_route: bool, + pub webhook_flow: bool, + pub webhook_script: bool, +} + +impl ServerToSet { + pub fn new(http_route: bool, webhook_flow: bool, webhook_script: bool) -> ServerToSet { + ServerToSet { http_route, webhook_flow, webhook_script } + } +} + +fn header_to_pascal_case(header: &str) -> String { + header + .split(|c: char| c == '-' || c == '_' || c == ' ') + .filter(|s| !s.is_empty()) + .map(|s| { + let mut chars = s.chars(); + match chars.next() { + Some(first) => { + first.to_ascii_uppercase().to_string() + + chars.as_str().to_ascii_lowercase().as_str() + } + None => String::new(), + } + }) + .collect::() +} + +#[derive(Debug, Default)] +struct SecuritySchemeToAdd { + basic_http: bool, + bearer_jwt: bool, + api_keys: Vec<(String, Value)>, +} + +fn generate_all_security_schemes(future_paths: &[FuturePath]) -> SecuritySchemeToAdd { + let mut to_add = SecuritySchemeToAdd::default(); + + let mut set = HashSet::new(); + for future_path in future_paths { + if !to_add.basic_http + && matches!(future_path.security_scheme, Some(SecurityScheme::BasicHttp)) + { + to_add.basic_http = true + } else if !to_add.bearer_jwt + && matches!(future_path.security_scheme, Some(SecurityScheme::BearerJwt)) + { + to_add.bearer_jwt = true + } else if let Some(SecurityScheme::ApiKey(api_key)) = future_path.security_scheme.as_ref() { + let pascal_case_header = header_to_pascal_case(&api_key); + + if !set.insert(pascal_case_header.clone()) { + continue; + } + + let scheme = serde_json::json!({ + "type": "apiKey", + "in": "header", + "name": api_key + }); + to_add.api_keys.push((pascal_case_header, scheme)); + } + } + + to_add +} + +fn generate_components(future_paths: &[FuturePath]) -> Map { + let mut components = Map::new(); + + if future_paths + .iter() + .any(|path| matches!(path.kind, Kind::Webhook(_))) + { + components.insert( + "parameters".to_owned(), + serde_json::json!({ + "PayloadParam": { + "name": "payload", + "in": "query", + "required": true, + "description": "A URL-safe base64-encoded JSON string payload.", + "schema": { + "type": "string" + } + } + }), + ); + } + + { + let mut security_scheme = Map::new(); + + let SecuritySchemeToAdd { basic_http, bearer_jwt, api_keys } = + generate_all_security_schemes(future_paths); + + if basic_http { + security_scheme.insert( + BASIC_HTTP_AUTH_SCHEME.to_owned(), + serde_json::json!({ + "type": "http", + "scheme": "basic" + }), + ); + } + + if bearer_jwt { + security_scheme.insert( + JWT_SECURITY_SCHEME.to_owned(), + serde_json::json!({ + "type": "http", + "scheme": "bearer", + "bearerFormat": "JWT" + }), + ); + } + + for (key, value) in api_keys { + security_scheme.insert(key, value); + } + components.insert("securitySchemes".to_owned(), Value::Object(security_scheme)); + } + + components.insert("requestBodies".to_owned(), serde_json::json!({ + DEFAULT_REQUEST_KEY: { + "description": "This route may or may not require a request body, but its structure and content type are unknown.", + "required": false, + "content": { + "application/json": {} + } + } + })); + + components.insert("responses".to_owned(), serde_json::json!({ + DEFAULT_ASYNC_RESPONSE_KEY: { + "description": "Returns a job ID as a UUID string.", + "content": { + "text/plain": { + "schema": { + "type": "string", + "format": "uuid", + "examples": [ "550e8400-e29b-41d4-a716-446655440000" ] + } + } + } + }, + DEFAULT_SYNC_RESPONSE_KEY: { + "description": "This route may return a response, but its structure and content type are unknown.", + "content": { + "application/octet-stream": {} + } + }, + + })); + + components +} + +pub fn generate_openapi_document( + info: Option<&Info>, + url: Option<&Url>, + paths: Vec, + format: Format, +) -> Result { + let mut openapi_doc: IndexMap<&'static str, Value> = IndexMap::new(); + + openapi_doc.insert("openapi", to_value(&DEFAULT_OPENAPI_GENERATED_VERSION)?); + openapi_doc.insert( + "info", + to_value(info.unwrap_or(&DEFAULT_OPENAPI_INFO_OBJECT))?, + ); + + openapi_doc.insert("components", Value::Object(generate_components(&paths))); + + openapi_doc.insert("paths", to_value(generate_paths(paths, url)?)?); + + let openapi_document = match format { + Format::YAML => serde_yml::to_string(&openapi_doc).map_err(|err| { + anyhow!( + "Could not generate OpenAPI document in YAML format: {}", + err + ) + })?, + Format::JSON => serde_json::to_string_pretty(&openapi_doc).map_err(|err| { + anyhow!( + "Could not generate OpenAPI document in JSON format: {}", + err + ) + })?, + }; + + Ok(openapi_document) +} + +#[allow(unused)] +#[derive(Debug, Deserialize)] +struct HttpRouteFilter { + folder_regex: String, + path_regex: String, + route_path_regex: String, +} + +#[derive(Debug, Deserialize)] +struct WebhookFilter { + user_or_folder_regex: String, + user_or_folder_regex_value: String, + path: String, + runnable_kind: RunnableKind, +} + +#[derive(Debug, Deserialize)] +struct GenerateOpenAPI { + info: Option, + url: Option, + #[serde(default, deserialize_with = "empty_as_none")] + http_route_filters: Option>, + #[serde(default, deserialize_with = "empty_as_none")] + webhook_filters: Option>, + #[serde(default)] + openapi_spec_format: Format, +} + +#[cfg(feature = "http_trigger")] +async fn http_routes_to_future_paths( + db: &DB, + user_db: UserDB, + authed: &ApiAuthed, + pg_pool: &mut PgConnection, + http_route_filters: Option<&[HttpRouteFilter]>, + w_id: &str, +) -> Result> { + let mut http_routes = Vec::new(); + + if let Some(http_route_filters) = http_route_filters { + let path_regex = http_route_filters + .iter() + .map(|filter| { + transform_to_minified_postgres_regex(&format!( + "f/{}/{}", + filter.folder_regex, filter.path_regex + )) + }) + .collect_vec(); + + let route_path_regex = http_route_filters + .iter() + .map(|filter| transform_to_minified_postgres_regex(&filter.route_path_regex)) + .collect_vec(); + + #[derive(Debug, Deserialize)] + struct MinifiedHttpTrigger { + route_path: String, + http_method: HttpMethod, + is_async: bool, + workspaced_route: bool, + summary: Option, + description: Option, + authentication_method: AuthenticationMethod, + authentication_resource_path: Option, + } + + http_routes = sqlx::query_as!( + MinifiedHttpTrigger, + r#" + SELECT + route_path, + http_method AS "http_method: _", + is_async, + workspaced_route, + summary, + description, + authentication_method AS "authentication_method: _", + authentication_resource_path + FROM + http_trigger + WHERE + path ~ ANY($1) AND + route_path ~ ANY($2) AND + workspace_id = $3 + "#, + &path_regex, + &route_path_regex, + &w_id + ) + .fetch_all(pg_pool) + .await?; + } + + let mut openapi_future_paths = Vec::with_capacity(http_routes.len()); + + for http_route in http_routes { + let auth_method = match http_route.authentication_method { + AuthenticationMethod::BasicHttp => Some(SecurityScheme::BasicHttp), + AuthenticationMethod::Windmill => Some(SecurityScheme::BearerJwt), + AuthenticationMethod::ApiKey => { + let resource_path = match http_route.authentication_resource_path { + Some(resource_path) => resource_path, + None => { + return Err(Error::BadRequest( + "Missing authentication resource path".to_string(), + )); + } + }; + + let api = try_get_resource_from_db_as::( + authed, + Some(user_db.clone()), + db, + &resource_path, + w_id, + ) + .await?; + + Some(SecurityScheme::ApiKey(api.api_key_header)) + } + _ => None, + }; + + let route_path = if http_route.workspaced_route { + format!("{}/{}", w_id, http_route.route_path.trim_start_matches('/')) + } else { + http_route.route_path.clone() + }; + + let method = match http_route.http_method { + HttpMethod::Get => Method::GET, + HttpMethod::Post => Method::POST, + HttpMethod::Put => Method::PUT, + HttpMethod::Patch => Method::PATCH, + HttpMethod::Delete => Method::DELETE, + }; + + let future_path = FuturePath::new( + route_path, + Kind::HttpRoute(HttpRouteConfig::new(method)), + Some(http_route.is_async), + http_route.summary, + http_route.description, + auth_method, + ); + + openapi_future_paths.push(future_path); + } + + Ok(openapi_future_paths) +} + +#[cfg(not(feature = "http_trigger"))] +async fn http_routes_to_future_paths( + _db: &DB, + _user_db: UserDB, + _authed: &ApiAuthed, + _pg_pool: &mut PgConnection, + _http_route_filters: Option<&[HttpRouteFilter]>, + _w_id: &str, +) -> Result> { + Ok(Vec::new()) +} + +async fn webhook_to_future_paths( + pg_pool: &mut PgConnection, + webhook_filters: Option<&[WebhookFilter]>, + w_id: &str, +) -> Result> { + let mut openapi_future_paths = Vec::new(); + if let Some(webhook_filters) = webhook_filters { + let mut script_webhook_filter = Vec::new(); + let mut flow_webhook_filter = Vec::new(); + + for webhook in webhook_filters { + let full_regex = transform_to_minified_postgres_regex(&format!( + "{}/{}/{}", + &webhook.user_or_folder_regex, &webhook.user_or_folder_regex_value, &webhook.path + )); + + match webhook.runnable_kind { + RunnableKind::Script => { + script_webhook_filter.push(full_regex); + } + RunnableKind::Flow => { + flow_webhook_filter.push(full_regex); + } + } + } + + #[derive(Debug, Deserialize, Clone, Hash)] + struct MinifiedWebhook { + path: String, + description: Option, + summary: Option, + } + + impl PartialEq for MinifiedWebhook { + fn eq(&self, other: &Self) -> bool { + self.path == other.path + } + } + + impl Eq for MinifiedWebhook {} + + let webhook_scripts = sqlx::query_as!( + MinifiedWebhook, + r#"SELECT + path, + summary, + description + FROM + script + WHERE + path ~ ANY($1) AND + workspace_id = $2 + "#, + &script_webhook_filter, + &w_id + ) + .fetch_all(&mut *pg_pool) + .await? + .into_iter() + .unique() + .collect_vec(); + + let webhook_flows = sqlx::query_as!( + MinifiedWebhook, + r#"SELECT + path, + summary, + description + FROM + flow + WHERE + path ~ ANY($1) AND + workspace_id = $2 + "#, + &flow_webhook_filter, + &w_id + ) + .fetch_all(&mut *pg_pool) + .await? + .into_iter() + .unique() + .collect_vec(); + + openapi_future_paths.reserve_exact(webhook_scripts.len() + webhook_flows.len()); + + for webhook in webhook_scripts { + openapi_future_paths.push(FuturePath::new( + webhook.path, + Kind::Webhook(WebhookConfig::new(RunnableKind::Script)), + None, + webhook.summary, + webhook.description, + Some(SecurityScheme::BearerJwt), + )); + } + + for webhook in webhook_flows { + openapi_future_paths.push(FuturePath::new( + webhook.path, + Kind::Webhook(WebhookConfig::new(RunnableKind::Flow)), + None, + webhook.summary, + webhook.description, + Some(SecurityScheme::BearerJwt), + )); + } + } + + Ok(openapi_future_paths) +} + +async fn generate_openapi_future_path( + db: &DB, + user_db: UserDB, + authed: &ApiAuthed, + http_route_filters: Option<&[HttpRouteFilter]>, + webhook_filters: Option<&[WebhookFilter]>, + w_id: &str, +) -> Result> { + if http_route_filters.is_none() && webhook_filters.is_none() { + return Err(Error::BadRequest( + "Expected http route filter and/or webhook filters".to_string(), + )); + } + + let mut tx = user_db.clone().begin(authed).await?; + + let mut openapi_future_paths = + http_routes_to_future_paths(db, user_db, authed, &mut tx, http_route_filters, w_id).await?; + + openapi_future_paths + .append(&mut webhook_to_future_paths(&mut tx, webhook_filters, w_id).await?); + + tx.commit().await?; + + if openapi_future_paths.is_empty() { + return Err(Error::NotFound( + "No match for the current filter".to_string(), + )); + } + + Ok(openapi_future_paths) +} + +async fn generate_openapi_spec( + Extension(authed): Extension, + Extension(db): Extension, + Extension(user_db): Extension, + Path(w_id): Path, + Json(generate_openapi): Json, +) -> Result { + let openapi_future_paths = generate_openapi_future_path( + &db, + user_db, + &authed, + generate_openapi.http_route_filters.as_deref(), + generate_openapi.webhook_filters.as_deref(), + &w_id, + ) + .await?; + + let openapi_document = generate_openapi_document( + generate_openapi.info.as_ref(), + generate_openapi.url.as_ref(), + openapi_future_paths, + generate_openapi.openapi_spec_format, + ); + + openapi_document +} + +async fn download_spec( + Extension(authed): Extension, + Extension(db): Extension, + Extension(user_db): Extension, + Path(w_id): Path, + Json(generate_openapi): Json, +) -> Result { + let openapi_future_paths = generate_openapi_future_path( + &db, + user_db, + &authed, + generate_openapi.http_route_filters.as_deref(), + generate_openapi.webhook_filters.as_deref(), + &w_id, + ) + .await?; + + let openapi_document = generate_openapi_document( + generate_openapi.info.as_ref(), + generate_openapi.url.as_ref(), + openapi_future_paths, + generate_openapi.openapi_spec_format, + )?; + + let response = Response::builder() + .status(StatusCode::OK) + .header( + header::CONTENT_TYPE, + HeaderValue::from_static("application/octet-stream"), + ) + .body(Body::from(openapi_document)) + .unwrap(); + + Ok(response) +} diff --git a/backend/windmill-api/src/postgres_triggers/handler.rs b/backend/windmill-api/src/postgres_triggers/handler.rs index 9beafadbf2..73e35c9e2f 100644 --- a/backend/windmill-api/src/postgres_triggers/handler.rs +++ b/backend/windmill-api/src/postgres_triggers/handler.rs @@ -15,14 +15,14 @@ use http::StatusCode; use itertools::Itertools; use pg_escape::{quote_identifier, quote_literal}; use quick_cache::sync::Cache; -use rust_postgres::types::Type; +use rust_postgres::{types::Type, Client}; use serde::{Deserialize, Deserializer, Serialize}; use sql_builder::{bind::Bind, SqlBuilder}; -use sqlx::{postgres::types::Oid, Connection, FromRow, PgConnection}; -use windmill_audit::{audit_ee::audit_log, ActionKind}; +use sqlx::FromRow; +use windmill_audit::{audit_oss::audit_log, ActionKind}; use windmill_common::{ db::UserDB, - error::{self, Error, JsonResult, Result}, + error::{self, to_anyhow, Error, JsonResult, Result}, utils::{empty_as_none, not_found_if_none, paginate, Pagination, StripPath}, worker::CLOUD_HOSTED, }; @@ -30,9 +30,10 @@ use windmill_git_sync::{handle_deployment_metadata, DeployedObject}; use super::{ check_if_valid_publication_for_postgres_version, create_logical_replication_slot, - create_pg_publication, drop_publication, generate_random_string, get_pg_connection, + create_pg_publication, drop_publication, generate_random_string, get_default_pg_connection, ERROR_PUBLICATION_NAME_NOT_EXISTS, }; +use anyhow::anyhow; use lazy_static::lazy_static; #[derive(FromRow, Serialize, Deserialize, Debug)] @@ -123,7 +124,7 @@ pub async fn test_postgres_connection( Json(test_postgres): Json, ) -> Result<()> { let connect_f = async { - get_pg_connection( + get_default_pg_connection( authed, Some(user_db), &db, @@ -267,15 +268,17 @@ impl PostgresPublicationReplication { } async fn check_if_logical_replication_slot_exist( - pg_connection: &mut PgConnection, + pg_connection: &mut Client, replication_slot_name: &str, ) -> Result { - let exists = sqlx::query("SELECT slot_name FROM pg_replication_slots where slot_name = $1") - .bind(&replication_slot_name) - .fetch_optional(pg_connection) - .await? - .is_some(); - Ok(exists) + let row = pg_connection + .query_opt( + "SELECT slot_name FROM pg_replication_slots WHERE slot_name = $1", + &[&replication_slot_name], + ) + .await + .map_err(to_anyhow)?; + Ok(row.is_some()) } async fn create_custom_slot_and_publication_inner( @@ -286,30 +289,31 @@ async fn create_custom_slot_and_publication_inner( w_id: &str, publication: &PublicationData, ) -> Result { - let mut pg_connection = get_pg_connection( + let mut pg_connection = get_default_pg_connection( authed.clone(), Some(user_db), &db, &postgres_resource_path, &w_id, ) - .await?; + .await + .map_err(to_anyhow)?; + + let tx = pg_connection.transaction().await.map_err(to_anyhow)?; - let mut tx = pg_connection.begin().await?; let publication_name = format!("windmill_trigger_{}", generate_random_string()); let replication_slot_name = publication_name.clone(); - - create_logical_replication_slot(&mut tx, &replication_slot_name).await?; - + + create_logical_replication_slot(tx.client(), &replication_slot_name).await?; create_pg_publication( - &mut tx, + &tx.client(), &publication_name, publication.table_to_track.as_deref(), &publication.transaction_to_track, ) .await?; - tx.commit().await?; + tx.commit().await.map_err(to_anyhow)?; Ok(PostgresPublicationReplication::new( publication_name, @@ -317,14 +321,13 @@ async fn create_custom_slot_and_publication_inner( )) } -pub async fn get_postgres_version_internal(pg_connection: &mut PgConnection) -> Result { - let postgres_version: String = sqlx::query_scalar("SHOW server_version;") - .fetch_one(&mut *pg_connection) +pub async fn get_postgres_version_internal(pg_connection: &Client) -> Result { + let row = pg_connection + .query_one("SHOW server_version;", &[]) .await - .map_err(|e| Error::Anyhow { - error: anyhow::anyhow!("Failed to retrieve PostgreSQL version: {}", e), - location: "postgres_triggers/handler.rs@379".to_string(), - })?; + .map_err(to_anyhow)?; + + let postgres_version: String = row.get(0); Ok(postgres_version) } @@ -335,16 +338,17 @@ pub async fn get_postgres_version( Extension(user_db): Extension, Path((w_id, postgres_resource_path)): Path<(String, String)>, ) -> Result { - let mut pg_connection = get_pg_connection( + let pg_connection = get_default_pg_connection( authed.clone(), Some(user_db), &db, &postgres_resource_path, &w_id, ) - .await?; + .await + .map_err(to_anyhow)?; - let postgres_version = get_postgres_version_internal(&mut pg_connection).await?; + let postgres_version = get_postgres_version_internal(&pg_connection).await?; Ok(postgres_version) } @@ -595,29 +599,37 @@ pub async fn list_slot_name( Extension(db): Extension, Path((w_id, postgres_resource_path)): Path<(String, String)>, ) -> Result>> { - let mut pg_connection = get_pg_connection( + let pg_connection: Client = get_default_pg_connection( authed.clone(), Some(user_db.clone()), &db, &postgres_resource_path, &w_id, ) - .await?; + .await + .map_err(to_anyhow)?; - let slots: Vec = sqlx::query_as( - r#" - SELECT - slot_name, - active - FROM - pg_replication_slots - WHERE - plugin = 'pgoutput' AND - slot_type = 'logical'; - "#, - ) - .fetch_all(&mut pg_connection) - .await?; + let rows = pg_connection + .query( + r#" + SELECT + slot_name, + active + FROM + pg_replication_slots + WHERE + plugin = 'pgoutput' AND + slot_type = 'logical'; + "#, + &[], + ) + .await + .map_err(to_anyhow)?; + + let slots = rows + .into_iter() + .map(|row| SlotList { slot_name: row.get("slot_name"), active: row.get("active") }) + .collect(); Ok(Json(slots)) } @@ -634,48 +646,51 @@ pub async fn create_slot( Path((w_id, postgres_resource_path)): Path<(String, String)>, Json(Slot { name }): Json, ) -> Result { - let mut pg_connection = get_pg_connection( + let pg_connection = get_default_pg_connection( authed.clone(), Some(user_db.clone()), &db, &postgres_resource_path, &w_id, ) - .await?; + .await + .map_err(to_anyhow)?; - create_logical_replication_slot(&mut pg_connection, &name).await?; + create_logical_replication_slot(&pg_connection, &name).await?; Ok(format!("Replication slot {} created!", name)) } -pub async fn drop_logical_replication_slot( - pg_connection: &mut PgConnection, - slot_name: &str, -) -> Result<()> { - let active_pid: Option = sqlx::query_scalar( - r#"SELECT - active_pid - FROM - pg_replication_slots - WHERE - slot_name = $1 - "#, - ) - .bind(&slot_name) - .fetch_optional(&mut *pg_connection) - .await? - .flatten(); +pub async fn drop_logical_replication_slot(pg_connection: &Client, slot_name: &str) -> Result<()> { + let row = pg_connection + .query_opt( + r#" + SELECT + active_pid + FROM + pg_replication_slots + WHERE + slot_name = $1 + "#, + &[&slot_name], + ) + .await + .map_err(to_anyhow)?; + + let active_pid = row.map(|r| r.get::<_, Option>(0)).flatten(); if let Some(pid) = active_pid { - sqlx::query("SELECT pg_terminate_backend($1)") - .bind(pid) - .execute(&mut *pg_connection) - .await?; + pg_connection + .execute("SELECT pg_terminate_backend($1)", &[&pid]) + .await + .map_err(to_anyhow)?; } - sqlx::query("SELECT pg_drop_replication_slot($1)") - .bind(&slot_name) - .execute(pg_connection) - .await?; + + pg_connection + .execute("SELECT pg_drop_replication_slot($1)", &[&slot_name]) + .await + .map_err(to_anyhow)?; + Ok(()) } @@ -686,17 +701,17 @@ pub async fn drop_slot_name( Path((w_id, postgres_resource_path)): Path<(String, String)>, Json(Slot { name }): Json, ) -> Result { - let mut pg_connection = - get_pg_connection(authed, Some(user_db), &db, &postgres_resource_path, &w_id).await?; + let pg_connection = + get_default_pg_connection(authed, Some(user_db), &db, &postgres_resource_path, &w_id) + .await + .map_err(to_anyhow)?; - drop_logical_replication_slot(&mut pg_connection, &name).await?; + drop_logical_replication_slot(&pg_connection, &name) + .await + .map_err(to_anyhow)?; Ok(format!("Replication slot {} deleted!", name)) } -#[derive(FromRow, Debug, Serialize)] -struct PublicationName { - publication_name: String, -} pub async fn list_database_publication( authed: ApiAuthed, @@ -704,23 +719,27 @@ pub async fn list_database_publication( Extension(db): Extension, Path((w_id, postgres_resource_path)): Path<(String, String)>, ) -> Result>> { - let mut pg_connection = get_pg_connection( + let pg_connection = get_default_pg_connection( authed.clone(), Some(user_db.clone()), &db, &postgres_resource_path, &w_id, ) - .await?; + .await + .map_err(to_anyhow)?; - let publication_names: Vec = - sqlx::query_as("SELECT pubname AS publication_name FROM pg_publication;") - .fetch_all(&mut pg_connection) - .await?; + let rows = pg_connection + .query( + "SELECT pubname AS publication_name FROM pg_publication;", + &[], + ) + .await + .map_err(to_anyhow)?; - let publications = publication_names - .iter() - .map(|publication| publication.publication_name.to_owned()) + let publications = rows + .into_iter() + .map(|row| row.get::<_, String>("publication_name")) .collect_vec(); Ok(Json(publications)) @@ -732,14 +751,15 @@ pub async fn get_publication_info( Extension(db): Extension, Path((w_id, publication_name, postgres_resource_path)): Path<(String, String, String)>, ) -> Result> { - let mut pg_connection = get_pg_connection( + let mut pg_connection = get_default_pg_connection( authed.clone(), Some(user_db.clone()), &db, &postgres_resource_path, &w_id, ) - .await?; + .await + .map_err(to_anyhow)?; let publication_data = get_publication_scope_and_transaction(&mut pg_connection, &publication_name).await; @@ -772,28 +792,29 @@ pub async fn create_publication( Path((w_id, publication_name, postgres_resource_path)): Path<(String, String, String)>, Json(publication_data): Json, ) -> Result { - let mut pg_connection = get_pg_connection( + let mut pg_connection = get_default_pg_connection( authed.clone(), Some(user_db.clone()), &db, &postgres_resource_path, &w_id, ) - .await?; + .await + .map_err(to_anyhow)?; let PublicationData { table_to_track, transaction_to_track } = publication_data; - let mut tx = pg_connection.begin().await?; + let tx = pg_connection.transaction().await.map_err(to_anyhow)?; create_pg_publication( - &mut tx, + tx.client(), &publication_name, table_to_track.as_deref(), &transaction_to_track, ) .await?; - tx.commit().await?; + tx.commit().await.map_err(to_anyhow)?; Ok(format!( "Publication {} successfully created!", @@ -807,14 +828,15 @@ pub async fn delete_publication( Extension(db): Extension, Path((w_id, publication_name, postgres_resource_path)): Path<(String, String, String)>, ) -> Result { - let mut pg_connection = get_pg_connection( + let mut pg_connection = get_default_pg_connection( authed.clone(), Some(user_db.clone()), &db, &postgres_resource_path, &w_id, ) - .await?; + .await + .map_err(to_anyhow)?; drop_publication(&mut pg_connection, &publication_name).await?; @@ -825,70 +847,67 @@ pub async fn delete_publication( } pub async fn update_pg_publication( - pg_connection: &mut PgConnection, + pg_connection: &Client, publication_name: &str, PublicationData { table_to_track, transaction_to_track }: PublicationData, all_table: Option, ) -> Result<()> { - let quoted_publication_name = quote_identifier(&publication_name); + let quoted_publication_name = quote_identifier(publication_name); let transaction_to_track_as_str = transaction_to_track.iter().join(","); match table_to_track { Some(ref relations) if !relations.is_empty() => { - //if all table is none it means that the publication do not exist in the database + // If all_table is None, the publication does not exist yet if all_table.unwrap_or(true) { - if all_table.is_some() { - drop_publication(pg_connection, &publication_name).await?; + if all_table.is_some_and(|all_table| all_table) { + drop_publication(pg_connection, publication_name) + .await + .map_err(to_anyhow)?; } create_pg_publication( pg_connection, - &publication_name, + publication_name, table_to_track.as_deref(), &transaction_to_track, ) - .await?; + .await + .map_err(to_anyhow)?; } else { let pg_14 = check_if_valid_publication_for_postgres_version( pg_connection, table_to_track.as_deref(), ) - .await?; + .await + .map_err(to_anyhow)?; - let mut query = String::from(""); + let mut query = format!("ALTER PUBLICATION {} SET ", quoted_publication_name); let mut first = true; - query.push_str("ALTER PUBLICATION "); - query.push_str("ed_publication_name); - query.push_str(" SET"); + for (i, schema) in relations.iter().enumerate() { if schema.table_to_track.is_empty() { - query.push_str(" TABLES IN SCHEMA "); - let quoted_schema = quote_identifier(&schema.schema_name); - query.push_str("ed_schema); + query.push_str("TABLES IN SCHEMA "); + query.push_str("e_identifier(&schema.schema_name)); } else { if pg_14 && first { - query.push_str(" TABLE ONLY "); - first = false + query.push_str("TABLE ONLY "); + first = false; } else if !pg_14 { - query.push_str(" TABLE ONLY "); + query.push_str("TABLE ONLY "); } + for (j, table) in schema.table_to_track.iter().enumerate() { let table_name = quote_identifier(&table.table_name); let schema_name = quote_identifier(&schema.schema_name); - let full_name = format!("{}.{}", &schema_name, &table_name); + let full_name = format!("{}.{}", schema_name, table_name); query.push_str(&full_name); + if let Some(columns) = table.columns_name.as_ref() { - query.push_str(" ("); - let columns = columns - .iter() - .map(|column| quote_identifier(column)) - .join(", "); - query.push_str(&columns); - query.push_str(") "); + let cols = + columns.iter().map(|col| quote_identifier(col)).join(", "); + query.push_str(&format!(" ({})", cols)); } if let Some(where_clause) = &table.where_clause { - query.push_str(" WHERE ("); - query.push_str(where_clause); - query.push(')'); + query.push_str(&format!(" WHERE ({})", where_clause)); } if j + 1 != schema.table_to_track.len() { @@ -896,39 +915,42 @@ pub async fn update_pg_publication( } } } - if i < relations.len() - 1 { - query.push(','); + + if i + 1 != relations.len() { + query.push_str(", "); } } - sqlx::query(&query).execute(&mut *pg_connection).await?; + pg_connection + .execute(&query, &[]) + .await + .map_err(to_anyhow)?; - let mut query = String::new(); - - query.push_str("ALTER PUBLICATION "); - query.push_str("ed_publication_name); - query.push_str(&format!( - " SET (publish = '{}');", - transaction_to_track_as_str - )); - - sqlx::query(&query).execute(pg_connection).await?; + let publish_query = format!( + "ALTER PUBLICATION {} SET (publish = '{}');", + quoted_publication_name, transaction_to_track_as_str + ); + pg_connection + .execute(&publish_query, &[]) + .await + .map_err(to_anyhow)?; } } _ => { - drop_publication(pg_connection, &publication_name).await?; - let query_to_execute = format!( - r#" - CREATE - PUBLICATION {} FOR ALL TABLES WITH (publish = '{}'); - "#, + drop_publication(pg_connection, publication_name) + .await + .map_err(to_anyhow)?; + let create_all_query = format!( + "CREATE PUBLICATION {} FOR ALL TABLES WITH (publish = '{}');", quoted_publication_name, transaction_to_track_as_str ); - sqlx::query(&query_to_execute) - .execute(pg_connection) - .await?; + pg_connection + .execute(&create_all_query, &[]) + .await + .map_err(to_anyhow)?; } - }; + } + Ok(()) } @@ -939,28 +961,32 @@ pub async fn alter_publication( Path((w_id, publication_name, postgres_resource_path)): Path<(String, String, String)>, Json(publication_data): Json, ) -> Result { - let mut pg_connection = get_pg_connection( + let mut pg_connection = get_default_pg_connection( authed.clone(), Some(user_db.clone()), &db, &postgres_resource_path, &w_id, ) - .await?; + .await + .map_err(to_anyhow)?; - let mut tx = pg_connection.begin().await?; - - let publication = get_publication_scope_and_transaction(&mut tx, &publication_name).await?; + let tx = pg_connection.transaction().await.map_err(to_anyhow)?; + + let publication = get_publication_scope_and_transaction(tx.client(), &publication_name) + .await + .map_err(to_anyhow)?; update_pg_publication( - &mut tx, + tx.client(), &publication_name, publication_data, publication.map(|publication| publication.0), ) - .await?; + .await + .map_err(to_anyhow)?; - tx.commit().await?; + tx.commit().await.map_err(to_anyhow)?; Ok(format!( "Publication {} updated with success", @@ -968,133 +994,126 @@ pub async fn alter_publication( )) } -async fn get_publication_scope_and_transaction( - pg_connection: &mut PgConnection, +pub async fn get_publication_scope_and_transaction( + pg_connection: &Client, publication_name: &str, ) -> Result)>> { - #[derive(Debug, Deserialize, FromRow)] - struct PublicationTransaction { - all_table: bool, - insert: bool, - update: bool, - delete: bool, - } + let row_opt = pg_connection + .query_opt( + r#" + SELECT + puballtables AS all_table, + pubinsert AS insert, + pubupdate AS update, + pubdelete AS delete + FROM + pg_publication + WHERE + pubname = $1 + "#, + &[&publication_name], + ) + .await + .map_err(to_anyhow)?; - let publication: Option = sqlx::query_as( - r#" - SELECT - puballtables AS all_table, - pubinsert AS insert, - pubupdate AS update, - pubdelete AS delete - FROM - pg_publication - WHERE - pubname = $1 - "#, - ) - .bind(publication_name) - .fetch_optional(&mut *pg_connection) - .await?; + let row = match row_opt { + Some(r) => r, + None => return Ok(None), + }; - if publication.is_none() { - return Ok(None); - } + let all_table: bool = row.get("all_table"); + let pub_insert: bool = row.get("insert"); + let pub_update: bool = row.get("update"); + let pub_delete: bool = row.get("delete"); let mut transaction_to_track = Vec::with_capacity(3); - - let publication = publication.unwrap(); - if publication.insert { + if pub_insert { transaction_to_track.push("insert".to_string()); } - if publication.update { + if pub_update { transaction_to_track.push("update".to_string()); } - if publication.delete { + if pub_delete { transaction_to_track.push("delete".to_string()); } - Ok(Some((publication.all_table, transaction_to_track))) + Ok(Some((all_table, transaction_to_track))) } -async fn get_tracked_relations( - pg_connection: &mut PgConnection, +pub async fn get_tracked_relations( + pg_connection: &Client, publication_name: &str, ) -> Result> { - #[derive(Debug, Deserialize, FromRow)] - struct PublicationData { - schema_name: Option, - table_name: Option, - #[serde(default)] - columns: Option>, - #[serde(default)] - where_clause: Option, - } - let pg_version = get_postgres_version_internal(pg_connection).await?; + let query = if pg_version.starts_with("14") { r#" - SELECT + SELECT schemaname AS schema_name, tablename AS table_name, NULL::text[] AS columns, NULL::text AS where_clause - FROM - pg_publication_tables - WHERE - pubname = $1; - "# + FROM + pg_publication_tables + WHERE + pubname = $1; + "# } else { r#" - SELECT + SELECT schemaname AS schema_name, tablename AS table_name, attnames AS columns, rowfilter AS where_clause - FROM - pg_publication_tables - WHERE - pubname = $1; - "# + FROM + pg_publication_tables + WHERE + pubname = $1; + "# }; - let publications: Vec = sqlx::query_as(query) - .bind(publication_name) - .fetch_all(&mut *pg_connection) - .await?; + let rows = pg_connection + .query(query, &[&publication_name]) + .await + .map_err(to_anyhow)?; let mut table_to_track: HashMap = HashMap::new(); - for publication in publications { - let schema_name = publication.schema_name.ok_or_else(|| Error::Anyhow { - error: anyhow::anyhow!( + for row in rows { + let schema_name: Option = row.get("schema_name"); + let table_name: Option = row.get("table_name"); + let columns: Option> = row.get("columns"); + let where_clause: Option = row.get("where_clause"); + + let schema_name = schema_name.ok_or_else::( || { + anyhow!( "Unexpected NULL `schema_name` in publication entry (pubname: `{}`). This should never happen unless PostgreSQL internals are corrupted.", publication_name, - ), - location: "postgres_triggers/handler.rs@1093".to_string(), - })?; + ).into() + } + )?; - let table_name = publication.table_name.ok_or_else(|| Error::Anyhow { - error: anyhow::anyhow!( + let table_name = table_name.ok_or_else::(|| { + anyhow!( "Unexpected NULL `table_name` for schema `{}` in publication `{}`. This should never happen unless PostgreSQL internals are corrupted.", schema_name, publication_name, - ), - location: "postgres_triggers/handler.rs@1102".to_string(), + ).into() })?; let entry = table_to_track.entry(schema_name.clone()); - let table_to_track = - TableToTrack::new(table_name, publication.where_clause, publication.columns); + let table_to_track = TableToTrack::new(table_name, where_clause, columns); + match entry { - Occupied(mut occuped) => { + std::collections::hash_map::Entry::Occupied(mut occuped) => { occuped.get_mut().add_new_table(table_to_track); } - Vacant(vacant) => { + std::collections::hash_map::Entry::Vacant(vacant) => { vacant.insert(Relations::new(schema_name, vec![table_to_track])); } } } + Ok(table_to_track.into_values().collect_vec()) } @@ -1161,41 +1180,48 @@ pub async fn update_postgres_trigger( publication, } = postgres_trigger; - let mut pg_connection = get_pg_connection( + let mut pg_connection = get_default_pg_connection( authed.clone(), Some(user_db.clone()), &db, &postgres_resource_path, &w_id, ) - .await?; + .await + .map_err(to_anyhow)?; let exists = check_if_logical_replication_slot_exist(&mut pg_connection, &replication_slot_name).await?; - let mut tx = pg_connection.begin().await?; + let tx = pg_connection.transaction().await.map_err(to_anyhow)?; if !exists { tracing::debug!( "Logical replication slot named: {} does not exists creating it...", &replication_slot_name ); - create_logical_replication_slot(&mut tx, &replication_slot_name).await?; + create_logical_replication_slot(tx.client(), &replication_slot_name) + .await + .map_err(to_anyhow)?; } if let Some(publication) = publication { let publication_data = - get_publication_scope_and_transaction(&mut tx, &publication_name).await?; + get_publication_scope_and_transaction(tx.client(), &publication_name) + .await + .map_err(to_anyhow)?; update_pg_publication( - &mut tx, + tx.client(), &publication_name, publication, publication_data.map(|publication| publication.0), ) - .await?; + .await + .map_err(to_anyhow)?; } - tx.commit().await?; + + tx.commit().await.map_err(to_anyhow)?; let mut tx = user_db.begin(&authed).await?; @@ -1415,53 +1441,40 @@ pub async fn create_template_script( Json(template_script): Json, ) -> Result { let TemplateScript { postgres_resource_path, relations, language } = template_script; - if relations.is_none() { - return Err(Error::BadRequest( - "You must at least choose schema to fetch table from".to_string(), - )); - } - let mut pg_connection = get_pg_connection( + let relations = match relations { + Some(r) => r, + None => return Err(anyhow!("You must at least choose schema to fetch table from").into()), + }; + + let pg_connection: Client = get_default_pg_connection( authed.clone(), Some(user_db.clone()), &db, &postgres_resource_path, &w_id, ) - .await?; + .await + .map_err(to_anyhow)?; - #[derive(Debug, FromRow, Deserialize)] - struct ColumnInfo { - table_schema: Option, - table_name: Option, - column_name: Option, - oid: Oid, - is_nullable: bool, - } - - let relations = relations.unwrap(); let mut schema_or_fully_qualified_name = Vec::with_capacity(relations.len()); - let mut columns_list = Vec::new(); + let mut columns_list = Vec::with_capacity(relations.len()); + for relation in relations { if !relation.table_to_track.is_empty() { - for table_to_track in relation.table_to_track { - let fully_qualified_name = - format!("{}.{}", &relation.schema_name, table_to_track.table_name); + for table in relation.table_to_track { + let fully_qualified_name = format!("{}.{}", relation.schema_name, table.table_name); schema_or_fully_qualified_name.push(quote_literal(&fully_qualified_name)); - - let columns = table_to_track + let columns = table .columns_name - .map(|columns| quote_literal(&columns.join(","))) - .or_else(|| Some("''".to_string())) - .unwrap(); - + .map(|c| quote_literal(&c.join(","))) + .unwrap_or_else(|| "''".to_string()); columns_list.push(columns); } - continue; + } else { + schema_or_fully_qualified_name.push(quote_literal(&relation.schema_name)); + columns_list.push("''".to_string()); } - - schema_or_fully_qualified_name.push(quote_literal(&relation.schema_name)); - columns_list.push(String::from("''")); } let tables_name = schema_or_fully_qualified_name.join(","); @@ -1481,8 +1494,7 @@ pub async fn create_template_script( WHEN tcm.column_list = '' THEN NULL ELSE string_to_array(tcm.column_list, ',') END AS columns - FROM - table_column_mapping tcm + FROM table_column_mapping tcm ) SELECT ns.nspname AS table_schema, @@ -1490,22 +1502,16 @@ pub async fn create_template_script( attr.attname AS column_name, attr.atttypid AS oid, attr.attnotnull AS is_nullable - FROM - pg_attribute attr - JOIN - pg_class cls - ON attr.attrelid = cls.oid - JOIN - pg_namespace ns - ON cls.relnamespace = ns.oid - JOIN - parsed_columns pc + FROM pg_attribute attr + JOIN pg_class cls ON attr.attrelid = cls.oid + JOIN pg_namespace ns ON cls.relnamespace = ns.oid + JOIN parsed_columns pc ON ns.nspname || '.' || cls.relname = pc.table_name OR ns.nspname = pc.table_name WHERE - attr.attnum > 0 -- Exclude system columns - AND NOT attr.attisdropped -- Exclude dropped columns - AND cls.relkind = 'r' -- Restrict to base tables + attr.attnum > 0 + AND NOT attr.attisdropped + AND cls.relkind = 'r' AND ( pc.columns IS NULL OR attr.attname = ANY(pc.columns) @@ -1514,51 +1520,52 @@ pub async fn create_template_script( tables_name, columns_list ); - let rows: Vec = sqlx::query_as(&query).fetch_all(&mut pg_connection).await?; - let mut mapper: HashMap>> = HashMap::new(); + let rows = pg_connection.query(&query, &[]).await.map_err(to_anyhow)?; + + let mut schema_map: HashMap>> = HashMap::new(); + + #[derive(Debug)] + struct ColumnInfo { + table_schema: String, + table_name: String, + column_name: String, + oid: u32, + is_nullable: bool, + } for row in rows { - let ColumnInfo { table_schema, table_name, column_name, oid, is_nullable } = row; - - let entry = mapper.entry(table_schema.unwrap()); + let info = ColumnInfo { + table_schema: row.get("table_schema"), + table_name: row.get("table_name"), + column_name: row.get("column_name"), + oid: row.get::<_, u32>("oid"), + is_nullable: row.get::<_, bool>("is_nullable"), + }; let mapped_info = - MappingInfo::new(column_name.unwrap(), Type::from_oid(oid.0), is_nullable); + MappingInfo::new(info.column_name, Type::from_oid(info.oid), info.is_nullable); - match entry { - Occupied(mut occupied) => { - let entry = occupied.get_mut().entry(table_name.unwrap()); - match entry { - Occupied(mut occuped) => { - let mapping_info = occuped.get_mut(); - mapping_info.push(mapped_info); - } - Vacant(vacant) => { - let mut mapping_info = Vec::with_capacity(10); - mapping_info.push(mapped_info); - vacant.insert(mapping_info); - } + match schema_map.entry(info.table_schema) { + Occupied(mut schema_entry) => match schema_entry.get_mut().entry(info.table_name) { + Occupied(mut table_entry) => { + table_entry.get_mut().push(mapped_info); } - } - Vacant(vacant) => { - let mut mapping_info = Vec::with_capacity(10); - mapping_info.push(mapped_info); - vacant.insert(HashMap::from([(table_name.unwrap(), mapping_info)])); + Vacant(v) => { + v.insert(vec![mapped_info]); + } + }, + Vacant(schema_vacant) => { + let mut table_map = HashMap::new(); + table_map.insert(info.table_name, vec![mapped_info]); + schema_vacant.insert(table_map); } } } - let mapper = Mapper::new(mapper, language); - - let create_template_id = |w_id: &str| -> String { - let uuid = uuid::Uuid::new_v4().to_string(); - let id = format!("{}-{}", &w_id, &uuid); - - id - }; - + let mapper = Mapper::new(schema_map, language); let template = mapper.get_template(); - let id = create_template_id(&w_id); + + let id = format!("{}-{}", w_id, uuid::Uuid::new_v4()); TEMPLATE.insert(id.clone(), template); @@ -1571,24 +1578,24 @@ pub async fn is_database_in_logical_level( Extension(db): Extension, Path((w_id, postgres_resource_path)): Path<(String, String)>, ) -> error::JsonResult { - let mut pg_connection = get_pg_connection( + let pg_connection = get_default_pg_connection( authed.clone(), Some(user_db.clone()), &db, &postgres_resource_path, &w_id, ) - .await?; + .await + .map_err(to_anyhow)?; - let wal_level: Option = sqlx::query_scalar("SHOW WAL_LEVEL;") - .fetch_optional(&mut pg_connection) - .await? - .flatten(); + let row_opt = pg_connection + .query_opt("SHOW wal_level;", &[]) + .await + .map_err(to_anyhow)?; - let is_logical = match wal_level.as_deref() { - Some("logical") => true, - _ => false, - }; + let wal_level: Option = row_opt.map(|row| row.get(0)); + + let is_logical = matches!(wal_level.as_deref(), Some("logical")); Ok(Json(is_logical)) } diff --git a/backend/windmill-api/src/postgres_triggers/mod.rs b/backend/windmill-api/src/postgres_triggers/mod.rs index 1cb0968f30..eff341436f 100644 --- a/backend/windmill-api/src/postgres_triggers/mod.rs +++ b/backend/windmill-api/src/postgres_triggers/mod.rs @@ -7,15 +7,13 @@ use crate::{ }; use chrono::Utc; use itertools::Itertools; -use pg_escape::{quote_identifier, quote_literal}; +use native_tls::{Certificate, TlsConnector}; +use pg_escape::quote_identifier; use rand::Rng; +use rust_postgres::{config::SslMode, Client, Config, NoTls}; +use rust_postgres_native_tls::MakeTlsConnector; use serde_json::value::RawValue; -use sqlx::{ - postgres::{PgConnectOptions, PgSslMode}, - Connection, PgConnection, -}; use std::collections::HashMap; -use std::str::FromStr; use axum::{ routing::{delete, get, post}, @@ -33,7 +31,7 @@ use handler::{ }; use windmill_common::{ db::UserDB, - error::{Error, Result}, + error::{to_anyhow, Error, Result}, utils::StripPath, }; mod bool; @@ -52,76 +50,144 @@ const ERROR_REPLICATION_SLOT_NOT_EXISTS: &str = r#"The replication slot associat const ERROR_PUBLICATION_NAME_NOT_EXISTS: &str = r#"The publication associated with this trigger no longer exists. Recreate a new publication or select an existing one in the advanced tab, or delete and recreate a new trigger"#; +fn build_tls_connector( + ssl_mode: SslMode, + root_certificate_pem: Option<&String>, +) -> Result> { + let get_tls_builder_for_verify = |root_certificate: Option<&String>| { + let mut builder = TlsConnector::builder(); + if let Some(root_certificate) = root_certificate { + let root_certificate_pem = + Certificate::from_pem(root_certificate.as_bytes()).map_err(to_anyhow)?; + builder.add_root_certificate(root_certificate_pem); + } + Ok::<_, Error>(builder) + }; + let connector = match ssl_mode { + SslMode::Disable => return Ok(None), + SslMode::Require | SslMode::Prefer => { + let mut builder = TlsConnector::builder(); + builder.danger_accept_invalid_certs(true); + builder.danger_accept_invalid_hostnames(true); + builder + } + + SslMode::VerifyCa => { + let mut builder = get_tls_builder_for_verify(root_certificate_pem)?; + builder.danger_accept_invalid_hostnames(true); + builder + } + + SslMode::VerifyFull => { + let builder = get_tls_builder_for_verify(root_certificate_pem)?; + builder + } + _ => unreachable!(), + }; + + Ok(Some(MakeTlsConnector::new( + connector.build().map_err(to_anyhow)?, + ))) +} + +pub async fn get_raw_postgres_connection( + database: &Postgres, + logical_mode: bool, +) -> Result { + let ssl_mode = match database.sslmode.as_ref() { + "disable" => SslMode::Disable, + "" | "prefer" | "allow" => SslMode::Prefer, + "require" => SslMode::Require, + "verify-ca" => SslMode::VerifyCa, + "verify-full" => SslMode::VerifyFull, + ssl_mode => { + return Err(Error::BadRequest( + format!("Invalid ssl mode for postgres: {}, please put a valid ssl_mode among the following available ssl mode: ['disable', 'allow', 'prefer', 'verify-ca', 'verify-full']", ssl_mode), + )) + } + }; + + let mut config = Config::new(); + config + .dbname(&database.dbname) + .host(&database.host) + .user(&database.user) + .ssl_mode(ssl_mode); + + if logical_mode { + config.replication_mode(rust_postgres::config::ReplicationMode::Logical); + } + + if let Some(port) = database.port { + config.port(port); + }; + + if !database.password.is_empty() { + config.password(&database.password); + } + + let connector = build_tls_connector(ssl_mode, database.root_certificate_pem.as_ref())?; + + let client = if let Some(connector) = connector { + let (client, connection) = config.connect(connector).await.map_err(to_anyhow)?; + tokio::spawn(async move { + if let Err(e) = connection.await { + tracing::debug!("{:#?}", e); + }; + tracing::info!("Successfully Connected into database"); + }); + client + } else { + let (client, connection) = config.connect(NoTls).await.map_err(to_anyhow)?; + tokio::spawn(async move { + if let Err(e) = connection.await { + tracing::debug!("{:#?}", e); + }; + tracing::info!("Successfully Connected into database"); + }); + client + }; + + Ok(client) +} + pub async fn get_pg_connection( authed: ApiAuthed, user_db: Option, db: &DB, postgres_resource_path: &str, w_id: &str, -) -> Result { + logical_mode: bool, +) -> Result { let database = - try_get_resource_from_db_as::(authed, user_db, db, postgres_resource_path, w_id) + try_get_resource_from_db_as::(&authed, user_db, db, postgres_resource_path, w_id) .await?; - Ok(get_raw_postgres_connection(&database).await?) + Ok(get_raw_postgres_connection(&database, logical_mode).await?) } -pub async fn get_raw_postgres_connection(db: &Postgres) -> Result { - let options = { - let sslmode = if !db.sslmode.is_empty() { - PgSslMode::from_str(&db.sslmode)? - } else { - PgSslMode::Prefer - }; - let options = { - let inner_options = PgConnectOptions::new() - .host(&db.host) - .database(&db.dbname) - .ssl_mode(sslmode) - .username(&db.user); - - if let Some(port) = db.port { - inner_options.port(port) - } else { - inner_options - } - }; - - let options = if let Some(root_certificate_pem) = &db.root_certificate_pem { - options.ssl_root_cert_from_pem(root_certificate_pem.as_bytes().to_vec()) - } else { - options - }; - - if !db.password.is_empty() { - options.password(&db.password) - } else { - options - } - }; - Ok(PgConnection::connect_with(&options).await?) +pub async fn get_default_pg_connection( + authed: ApiAuthed, + user_db: Option, + db: &DB, + postgres_resource_path: &str, + w_id: &str, +) -> Result { + get_pg_connection(authed, user_db, db, postgres_resource_path, w_id, false).await } -pub async fn create_logical_replication_slot( - pg_connection: &mut PgConnection, - name: &str, -) -> Result<()> { - let query = format!( - r#" - SELECT - * - FROM - pg_create_logical_replication_slot({}, 'pgoutput');"#, - quote_literal(&name) - ); - - sqlx::query(&query).execute(pg_connection).await?; - +pub async fn create_logical_replication_slot(tx: &Client, slot_name: &str) -> Result<()> { + tx.execute( + &format!("SELECT * FROM pg_create_logical_replication_slot($1, 'pgoutput')"), + &[&slot_name], + ) + .await + .map_err(to_anyhow)?; Ok(()) } async fn check_if_valid_publication_for_postgres_version( - pg_connection: &mut PgConnection, + pg_connection: &Client, table_to_track: Option<&[Relations]>, ) -> Result { let postgres_version = get_postgres_version_internal(pg_connection).await?; @@ -155,7 +221,7 @@ async fn check_if_valid_publication_for_postgres_version( } pub async fn create_pg_publication( - pg_connection: &mut PgConnection, + pg_connection: &Client, publication_name: &str, table_to_track: Option<&[Relations]>, transaction_to_track: &[String], @@ -177,7 +243,7 @@ pub async fn create_pg_publication( } else { if pg_14 && first { query.push_str(" TABLE ONLY "); - first = false; + first = false } else if !pg_14 { query.push_str(" TABLE ONLY "); } @@ -224,20 +290,22 @@ pub async fn create_pg_publication( query.push_str("');"); } - sqlx::query(&query).execute(pg_connection).await?; - + pg_connection + .execute(&query, &[]) + .await + .map_err(to_anyhow)?; Ok(()) } -pub async fn drop_publication( - pg_connection: &mut PgConnection, - publication_name: &str, -) -> Result<()> { +pub async fn drop_publication(pg_connection: &Client, publication_name: &str) -> Result<()> { let mut query = String::from("DROP PUBLICATION IF EXISTS "); let quoted_publication_name = quote_identifier(publication_name); query.push_str("ed_publication_name); - sqlx::query(&query).execute(pg_connection).await?; + pg_connection + .execute(&query, &[]) + .await + .map_err(to_anyhow)?; Ok(()) } diff --git a/backend/windmill-api/src/postgres_triggers/trigger.rs b/backend/windmill-api/src/postgres_triggers/trigger.rs index b16862992e..27313879b2 100644 --- a/backend/windmill-api/src/postgres_triggers/trigger.rs +++ b/backend/windmill-api/src/postgres_triggers/trigger.rs @@ -19,25 +19,27 @@ use crate::{ use bytes::{BufMut, Bytes, BytesMut}; use chrono::TimeZone; use futures::{pin_mut, SinkExt, StreamExt}; -use native_tls::{Certificate, TlsConnector}; use pg_escape::{quote_identifier, quote_literal}; use rand::seq::SliceRandom; -use rust_postgres::{config::SslMode, Client, Config, CopyBothDuplex, NoTls, SimpleQueryMessage}; -use rust_postgres_native_tls::MakeTlsConnector; +use rust_postgres::{Client, CopyBothDuplex, SimpleQueryMessage}; use serde::Deserialize; use serde_json::value::RawValue; use sqlx::types::Json as SqlxJson; use windmill_common::{ - db::UserDB, error, triggers::TriggerKind, utils::report_critical_error, worker::to_raw_value, + db::UserDB, + error::{self, to_anyhow}, + triggers::TriggerKind, + utils::report_critical_error, + worker::to_raw_value, INSTANCE_NAME, }; use super::{ - drop_publication, get_pg_connection, + drop_publication, get_default_pg_connection, get_raw_postgres_connection, handler::{drop_logical_replication_slot, Postgres, PostgresTrigger}, replication_message::PrimaryKeepAliveBody, - ERROR_PUBLICATION_NAME_NOT_EXISTS, ERROR_REPLICATION_SLOT_NOT_EXISTS, + Error, ERROR_PUBLICATION_NAME_NOT_EXISTS, ERROR_REPLICATION_SLOT_NOT_EXISTS, }; pub struct LogicalReplicationSettings { @@ -69,109 +71,11 @@ impl RowExist for Vec { } } -#[derive(thiserror::Error, Debug)] -enum Error { - #[error("Error from database: {0}")] - Postgres(#[from] rust_postgres::Error), - #[error("Error : {0}")] - Common(#[from] windmill_common::error::Error), - #[error("Tls Error: {0}")] - Tls(#[from] native_tls::Error), -} - -fn build_tls_connector( - ssl_mode: SslMode, - root_certificate_pem: Option<&String>, -) -> Result, Error> { - let get_tls_builder_for_verify = |root_certificate: Option<&String>| { - let mut builder = TlsConnector::builder(); - if let Some(root_certificate) = root_certificate { - let root_certificate_pem = Certificate::from_pem(root_certificate.as_bytes()).map_err(|e| { - Error::Common(error::Error::BadConfig(format!("Invalid Certs: {e:#}"))) - })?; - builder.add_root_certificate(root_certificate_pem); - } - Ok::<_, Error>(builder) - }; - let connector = match ssl_mode { - SslMode::Disable => return Ok(None), - SslMode::Require | SslMode::Prefer => { - let mut builder = TlsConnector::builder(); - builder.danger_accept_invalid_certs(true); - builder.danger_accept_invalid_hostnames(true); - builder - } - - SslMode::VerifyCa => { - let mut builder = get_tls_builder_for_verify(root_certificate_pem)?; - builder.danger_accept_invalid_hostnames(true); - builder - } - - SslMode::VerifyFull => { - let builder = get_tls_builder_for_verify(root_certificate_pem)?; - builder - } - _ => unreachable!(), - }; - - Ok(Some(MakeTlsConnector::new(connector.build()?))) -} - pub struct PostgresSimpleClient(Client); impl PostgresSimpleClient { async fn new(database: &Postgres) -> Result { - let ssl_mode = match database.sslmode.as_ref() { - "disable" => SslMode::Disable, - "" | "prefer" | "allow" => SslMode::Prefer, - "require" => SslMode::Require, - "verify-ca" => SslMode::VerifyCa, - "verify-full" => SslMode::VerifyFull, - ssl_mode => { - return Err(Error::Common(windmill_common::error::Error::BadRequest( - format!("Invalid ssl mode for postgres: {}, please put a valid ssl_mode among the following avalible ssl mode: ['disable', 'allow', 'prefer', 'verify-ca', 'verify-full']", ssl_mode), - ))) - } - }; - - let mut config = Config::new(); - config - .dbname(&database.dbname) - .host(&database.host) - .user(&database.user) - .ssl_mode(ssl_mode) - .replication_mode(rust_postgres::config::ReplicationMode::Logical); - - if let Some(port) = database.port { - config.port(port); - }; - - if !database.password.is_empty() { - config.password(&database.password); - } - - let connector = build_tls_connector(ssl_mode, database.root_certificate_pem.as_ref())?; - - let client = if let Some(connector) = connector { - let (client, connection) = config.connect(connector).await?; - tokio::spawn(async move { - if let Err(e) = connection.await { - tracing::debug!("{:#?}", e); - }; - tracing::info!("Successfully Connected into database"); - }); - client - } else { - let (client, connection) = config.connect(NoTls).await?; - tokio::spawn(async move { - if let Err(e) = connection.await { - tracing::debug!("{:#?}", e); - }; - tracing::info!("Successfully Connected into database"); - }); - client - }; + let client = get_raw_postgres_connection(database, true).await?; Ok(PostgresSimpleClient(client)) } @@ -202,7 +106,8 @@ impl PostgresSimpleClient { Ok(( self.0 .copy_both_simple::(query.as_str()) - .await?, + .await + .map_err(to_anyhow)?, LogicalReplicationSettings::new(false), )) } @@ -512,7 +417,7 @@ impl PostgresConfig { }; let database = try_get_resource_from_db_as::( - authed, + &authed, Some(UserDB::new(db.clone())), &db, postgres_resource_path, @@ -527,12 +432,13 @@ impl PostgresConfig { "SELECT pubname FROM pg_publication WHERE pubname = {}", quote_literal(&publication_name) )) - .await?; + .await + .map_err(to_anyhow)?; if !publication.row_exist() { - return Err(Error::Common(error::Error::BadConfig( + return Err(Error::BadConfig( ERROR_PUBLICATION_NAME_NOT_EXISTS.to_string(), - ))); + )); } let replication_slot = client @@ -540,17 +446,19 @@ impl PostgresConfig { "SELECT slot_name FROM pg_replication_slots WHERE slot_name = {}", quote_literal(&replication_slot_name) )) - .await?; + .await + .map_err(to_anyhow)?; if !replication_slot.row_exist() { - return Err(Error::Common(error::Error::BadConfig( + return Err(Error::BadConfig( ERROR_REPLICATION_SLOT_NOT_EXISTS.to_string(), - ))); + )); } let (logical_replication_stream, logical_replication_settings) = client .get_logical_replication_stream(&publication_name, &replication_slot_name) - .await?; + .await + .map_err(to_anyhow)?; Ok((logical_replication_stream, logical_replication_settings)) } @@ -585,7 +493,7 @@ impl PostgresConfig { let user_db = UserDB::new(db.clone()); - let mut pg_connection = get_pg_connection( + let mut pg_connection = get_default_pg_connection( authed.clone(), Some(user_db.clone()), &db, diff --git a/backend/windmill-api/src/raw_apps.rs b/backend/windmill-api/src/raw_apps.rs index bb2763b73d..3a895bab97 100644 --- a/backend/windmill-api/src/raw_apps.rs +++ b/backend/windmill-api/src/raw_apps.rs @@ -22,7 +22,7 @@ use serde::{Deserialize, Serialize}; use sql_builder::{bind::Bind, SqlBuilder}; use sqlx::FromRow; use std::str; -use windmill_audit::audit_ee::audit_log; +use windmill_audit::audit_oss::audit_log; use windmill_audit::ActionKind; use windmill_common::{ apps::ListAppQuery, diff --git a/backend/windmill-api/src/resources.rs b/backend/windmill-api/src/resources.rs index 8a2fab0434..14fdfef536 100644 --- a/backend/windmill-api/src/resources.rs +++ b/backend/windmill-api/src/resources.rs @@ -26,7 +26,7 @@ use serde_json::{value::RawValue, Value}; use sql_builder::{bind::Bind, quote, SqlBuilder}; use sqlx::{FromRow, Postgres, Transaction}; use uuid::Uuid; -use windmill_audit::audit_ee::{audit_log, AuditAuthor}; +use windmill_audit::audit_oss::{audit_log, AuditAuthor}; use windmill_audit::ActionKind; use windmill_common::{ db::UserDB, @@ -1217,7 +1217,7 @@ async fn update_resource_type( ) ))] pub async fn try_get_resource_from_db_as( - authed: ApiAuthed, + authed: &ApiAuthed, user_db: Option, db: &DB, resource_path: &str, diff --git a/backend/windmill-api/src/saml_ee.rs b/backend/windmill-api/src/saml_oss.rs similarity index 75% rename from backend/windmill-api/src/saml_ee.rs rename to backend/windmill-api/src/saml_oss.rs index b3f1d4653c..0f4f4aa7f6 100644 --- a/backend/windmill-api/src/saml_ee.rs +++ b/backend/windmill-api/src/saml_oss.rs @@ -7,18 +7,27 @@ */ #![allow(non_snake_case)] +#[cfg(feature = "private")] +#[allow(unused)] +pub use crate::saml_ee::*; + +#[cfg(not(feature = "private"))] use axum::{routing::post, Router}; +#[cfg(not(feature = "private"))] pub struct ServiceProviderExt(); +#[cfg(not(feature = "private"))] pub async fn build_sp_extension() -> anyhow::Result { return Ok(ServiceProviderExt()); } +#[cfg(not(feature = "private"))] pub fn global_service() -> Router { Router::new().route("/acs", post(acs)) } +#[cfg(not(feature = "private"))] pub async fn acs() -> String { // Implementation is not open source as it is a Windmill Enterprise Edition feature "SAML available only in enterprise version".to_string() diff --git a/backend/windmill-api/src/schedule.rs b/backend/windmill-api/src/schedule.rs index 90d91c846c..3cb13798d9 100644 --- a/backend/windmill-api/src/schedule.rs +++ b/backend/windmill-api/src/schedule.rs @@ -22,7 +22,7 @@ use serde::{Deserialize, Serialize}; use sql_builder::{prelude::Bind, SqlBuilder}; use sqlx::{Postgres, Transaction}; use std::str::FromStr; -use windmill_audit::audit_ee::audit_log; +use windmill_audit::audit_oss::audit_log; use windmill_audit::ActionKind; use windmill_common::{ db::UserDB, diff --git a/backend/windmill-api/src/scim_ee.rs b/backend/windmill-api/src/scim_oss.rs similarity index 72% rename from backend/windmill-api/src/scim_ee.rs rename to backend/windmill-api/src/scim_oss.rs index f11097f874..5210411466 100644 --- a/backend/windmill-api/src/scim_ee.rs +++ b/backend/windmill-api/src/scim_oss.rs @@ -1,3 +1,7 @@ +#[cfg(feature = "private")] +#[allow(unused)] +pub use crate::scim_ee::*; + /* * Author: Ruben Fiszel * Copyright: Windmill Labs, Inc 2023 @@ -6,17 +10,22 @@ * LICENSE-AGPL for a copy of the license. */ +#[cfg(not(feature = "private"))] use axum::{middleware::Next, response::Response, routing::get, Router}; +#[cfg(not(feature = "private"))] use hyper::Request; +#[cfg(not(feature = "private"))] pub fn global_service() -> Router { Router::new().route("/ee", get(ee)) } +#[cfg(not(feature = "private"))] pub async fn ee() -> String { return "Enterprise Edition".to_string(); } +#[cfg(not(feature = "private"))] pub async fn has_scim_token(_request: Request, _next: Next) -> Response { //Not implemented in open-source version todo!() diff --git a/backend/windmill-api/src/scripts.rs b/backend/windmill-api/src/scripts.rs index a827fa5079..5e49dc4d1c 100644 --- a/backend/windmill-api/src/scripts.rs +++ b/backend/windmill-api/src/scripts.rs @@ -38,7 +38,7 @@ use std::{ hash::{Hash, Hasher}, sync::Arc, }; -use windmill_audit::audit_ee::audit_log; +use windmill_audit::audit_oss::audit_log; use windmill_audit::ActionKind; use windmill_worker::process_relative_imports; @@ -206,7 +206,6 @@ async fn list_scripts( Query(lq): Query, ) -> JsonResult> { let (per_page, offset) = paginate(pagination); - let mut sqlb = SqlBuilder::select_from("script as o") .fields(&[ "hash", @@ -321,6 +320,16 @@ async fn list_scripts( .fields(&["dm.deployment_msg"]); } + if let Some(languages) = lq.languages { + sqlb.and_where_in( + "language", + &languages + .iter() + .map(|language| quote(language.as_str())) + .collect_vec(), + ); + } + let sql = sqlb.sql().map_err(|e| Error::internal_err(e.to_string()))?; let mut tx = user_db.begin(&authed).await?; let rows = sqlx::query_as::<_, ListableScript>(&sql) @@ -1280,6 +1289,11 @@ async fn raw_script_by_path_unpinned( raw_script_by_path_internal(path, user_db, db, authed, w_id, true).await } +lazy_static::lazy_static! { + static ref DEBUG_RAW_SCRIPT_ENDPOINTS: bool = + std::env::var("DEBUG_RAW_SCRIPT_ENDPOINTS").is_ok(); +} + async fn raw_script_by_path_internal( path: StripPath, user_db: UserDB, @@ -1332,6 +1346,26 @@ async fn raw_script_by_path_internal( "Script {path} exists but {} does not have permissions to access it", authed.username ))); + } else { + if *DEBUG_RAW_SCRIPT_ENDPOINTS { + let other_script_o = sqlx::query_scalar!( + "SELECT path FROM script WHERE workspace_id = $1 AND archived = false", + w_id + ) + .fetch_all(&db) + .await?; + let other_script_archived = sqlx::query_scalar!( + "SELECT distinct(path) FROM script WHERE workspace_id = $1 AND archived = true", + w_id + ) + .fetch_all(&db) + .await?; + tracing::warn!( + "Script {path} does not exist in workspace {w_id} but these paths do, non-archived: {:?} | archived: {:?}", + other_script_o.join(", "), + other_script_archived.join(", ") + ) + } } } diff --git a/backend/windmill-api/src/settings.rs b/backend/windmill-api/src/settings.rs index b589713b91..9dfb687e51 100644 --- a/backend/windmill-api/src/settings.rs +++ b/backend/windmill-api/src/settings.rs @@ -10,7 +10,7 @@ use std::time::Duration; use crate::{ db::{ApiAuthed, DB}, - ee::validate_license_key, + ee_oss::validate_license_key, utils::{generate_instance_username_for_all_users, require_super_admin}, HTTP_CLIENT, }; @@ -29,9 +29,9 @@ use crate::utils::require_devops_role; use serde::Deserialize; #[cfg(feature = "enterprise")] -use windmill_common::ee::{send_critical_alert, CriticalAlertKind, CriticalErrorChannel}; +use windmill_common::ee_oss::{send_critical_alert, CriticalAlertKind, CriticalErrorChannel}; use windmill_common::{ - email_ee::send_email, + email_oss::send_email, error::{self, JsonResult, Result}, global_settings::{ AUTOMATE_USERNAME_CREATION_SETTING, CRITICAL_ALERT_MUTE_UI_SETTING, EMAIL_DOMAIN_SETTING, @@ -326,10 +326,10 @@ async fn list_global_settings() -> JsonResult { pub async fn send_stats(Extension(db): Extension, authed: ApiAuthed) -> Result { require_super_admin(&db, &authed.email).await?; - windmill_common::stats_ee::send_stats( + windmill_common::stats_oss::send_stats( &HTTP_CLIENT, &db, - windmill_common::stats_ee::SendStatsReason::Manual, + windmill_common::stats_oss::SendStatsReason::Manual, ) .await?; @@ -390,11 +390,11 @@ pub async fn renew_license_key( authed: ApiAuthed, ) -> Result { require_super_admin(&db, &authed.email).await?; - let result = windmill_common::ee::renew_license_key( + let result = windmill_common::ee_oss::renew_license_key( &HTTP_CLIENT, &db, license_key, - windmill_common::ee::RenewReason::Manual, + windmill_common::ee_oss::RenewReason::Manual, ) .await; @@ -424,7 +424,7 @@ pub async fn create_customer_portal_session( Query(LicenseQuery { license_key }): Query, ) -> Result { let url = - windmill_common::ee::create_customer_portal_session(&HTTP_CLIENT, license_key).await?; + windmill_common::ee_oss::create_customer_portal_session(&HTTP_CLIENT, license_key).await?; return Ok(url); } diff --git a/backend/windmill-api/src/smtp_server_ee.rs b/backend/windmill-api/src/smtp_server_oss.rs similarity index 69% rename from backend/windmill-api/src/smtp_server_ee.rs rename to backend/windmill-api/src/smtp_server_oss.rs index 48e274f6a1..7d2b72c7f1 100644 --- a/backend/windmill-api/src/smtp_server_ee.rs +++ b/backend/windmill-api/src/smtp_server_oss.rs @@ -1,7 +1,15 @@ +#[cfg(feature = "private")] +#[allow(unused)] +pub use crate::smtp_server_ee::*; + +#[cfg(not(feature = "private"))] use crate::{auth::AuthCache, db::DB}; +#[cfg(not(feature = "private"))] use std::{net::SocketAddr, sync::Arc}; +#[cfg(not(feature = "private"))] use windmill_common::db::UserDB; +#[cfg(not(feature = "private"))] pub struct SmtpServer { pub auth_cache: Arc, pub db: DB, @@ -9,6 +17,7 @@ pub struct SmtpServer { pub base_internal_url: String, } +#[cfg(not(feature = "private"))] impl SmtpServer { pub async fn start_listener_thread(self: Arc, _addr: SocketAddr) -> anyhow::Result<()> { let _ = self.auth_cache; diff --git a/backend/windmill-api/src/sqs_triggers_ee.rs b/backend/windmill-api/src/sqs_triggers_oss.rs similarity index 75% rename from backend/windmill-api/src/sqs_triggers_ee.rs rename to backend/windmill-api/src/sqs_triggers_oss.rs index da9b288f6b..6e01641a5e 100644 --- a/backend/windmill-api/src/sqs_triggers_ee.rs +++ b/backend/windmill-api/src/sqs_triggers_oss.rs @@ -1,18 +1,28 @@ +#[cfg(feature = "private")] +#[allow(unused)] +pub use crate::sqs_triggers_ee::*; + +#[cfg(not(feature = "private"))] use crate::db::DB; +#[cfg(not(feature = "private"))] use axum::Router; +#[cfg(not(feature = "private"))] use serde::{Deserialize, Serialize}; +#[cfg(not(feature = "private"))] use windmill_common::auth::aws::AwsAuthResourceType; - +#[cfg(not(feature = "private"))] pub fn workspaced_service() -> Router { Router::new() } +#[cfg(not(feature = "private"))] pub fn start_sqs(_db: DB, mut _killpill_rx: tokio::sync::broadcast::Receiver<()>) -> () { // implementation is not open source } #[derive(Debug, Clone, Deserialize, Serialize)] +#[cfg(not(feature = "private"))] pub struct SqsTrigger { pub queue_url: String, pub aws_auth_resource_type: AwsAuthResourceType, @@ -30,4 +40,4 @@ pub struct SqsTrigger { pub server_id: Option, pub last_server_ping: Option>, pub enabled: bool, -} \ No newline at end of file +} diff --git a/backend/windmill-api/src/stripe_ee.rs b/backend/windmill-api/src/stripe_ee.rs deleted file mode 100644 index 6934cad3bc..0000000000 --- a/backend/windmill-api/src/stripe_ee.rs +++ /dev/null @@ -1,5 +0,0 @@ -use axum::Router; - -pub fn add_stripe_routes(router: Router) -> Router { - return router; -} diff --git a/backend/windmill-api/src/stripe_oss.rs b/backend/windmill-api/src/stripe_oss.rs new file mode 100644 index 0000000000..206a78dc53 --- /dev/null +++ b/backend/windmill-api/src/stripe_oss.rs @@ -0,0 +1,11 @@ +#[cfg(feature = "private")] +#[allow(unused)] +pub use crate::stripe_ee::*; + +#[cfg(not(feature = "private"))] +use axum::Router; + +#[cfg(not(feature = "private"))] +pub fn add_stripe_routes(router: Router) -> Router { + return router; +} diff --git a/backend/windmill-api/src/teams_approvals_ee.rs b/backend/windmill-api/src/teams_approvals_oss.rs similarity index 51% rename from backend/windmill-api/src/teams_approvals_ee.rs rename to backend/windmill-api/src/teams_approvals_oss.rs index 05698b98e3..df5826aa71 100644 --- a/backend/windmill-api/src/teams_approvals_ee.rs +++ b/backend/windmill-api/src/teams_approvals_oss.rs @@ -1,7 +1,14 @@ +#[cfg(feature = "private")] +#[allow(unused)] +pub use crate::teams_approvals_ee::*; + +#[cfg(not(feature = "private"))] use hyper::StatusCode; +#[cfg(not(feature = "private"))] use windmill_common::error::Error; +#[cfg(not(feature = "private"))] pub async fn request_teams_approval() -> Result { Err(Error::InternalErr("enterprise feature only".to_string())) -} \ No newline at end of file +} diff --git a/backend/windmill-api/src/teams_ee.rs b/backend/windmill-api/src/teams_oss.rs similarity index 70% rename from backend/windmill-api/src/teams_ee.rs rename to backend/windmill-api/src/teams_oss.rs index 46cbe72059..95d4883690 100644 --- a/backend/windmill-api/src/teams_ee.rs +++ b/backend/windmill-api/src/teams_oss.rs @@ -1,39 +1,50 @@ -use http::status::StatusCode; -#[cfg(feature = "enterprise")] +#[cfg(feature = "private")] +#[allow(unused)] +pub use crate::teams_ee::*; + +#[cfg(all(feature = "enterprise", not(feature = "private")))] use axum::Router; +#[cfg(not(feature = "private"))] +use http::status::StatusCode; +#[cfg(not(feature = "private"))] use windmill_common::error::Error; +#[cfg(not(feature = "private"))] pub async fn edit_teams_command() -> Result { return Err(Error::BadRequest( "Teams only available on enterprise".to_string(), )); } +#[cfg(not(feature = "private"))] pub async fn workspaces_list_available_teams_ids() -> Result { return Err(Error::BadRequest( "Teams only available on enterprise".to_string(), )); } +#[cfg(not(feature = "private"))] pub async fn connect_teams() -> Result { return Err(Error::BadRequest( "Teams only available on enterprise".to_string(), )); } +#[cfg(not(feature = "private"))] pub async fn run_teams_message_test_job() -> Result { return Err(Error::BadRequest( "Teams only available on enterprise".to_string(), )); } +#[cfg(not(feature = "private"))] pub async fn workspaces_list_available_teams_channels() -> Result { return Err(Error::BadRequest( "Teams only available on enterprise".to_string(), )); } -#[cfg(feature = "enterprise")] +#[cfg(all(feature = "enterprise", not(feature = "private")))] pub fn teams_service() -> Router { Router::new() -} \ No newline at end of file +} diff --git a/backend/windmill-api/src/users.rs b/backend/windmill-api/src/users.rs index 5bc7ce9c4d..58e9d2d849 100644 --- a/backend/windmill-api/src/users.rs +++ b/backend/windmill-api/src/users.rs @@ -42,7 +42,7 @@ use sqlx::FromRow; use time::OffsetDateTime; use tower_cookies::{Cookie, Cookies}; use tracing::Instrument; -use windmill_audit::audit_ee::{audit_log, AuditAuthor}; +use windmill_audit::audit_oss::{audit_log, AuditAuthor}; use windmill_audit::ActionKind; use windmill_common::auth::fetch_authed_from_permissioned_as; use windmill_common::global_settings::AUTOMATE_USERNAME_CREATION_SETTING; @@ -542,7 +542,7 @@ async fn list_users_as_super_admin( let rows = if active_only.is_some_and(|x| x) { sqlx::query_as!( GlobalUserInfo, - "WITH active_users AS (SELECT distinct username as email FROM audit WHERE timestamp > NOW() - INTERVAL '1 month' AND (operation = 'users.login' OR operation = 'oauth.login')), + "WITH active_users AS (SELECT distinct username as email FROM audit WHERE timestamp > NOW() - INTERVAL '1 month' AND (operation = 'users.login' OR operation = 'oauth.login' OR operation = 'users.token.refresh')), authors as (SELECT distinct email FROM usr WHERE usr.operator IS false) SELECT email, email NOT IN (SELECT email FROM authors) as operator_only, login_type::text, verified, super_admin, devops, name, company, username FROM password @@ -1508,7 +1508,7 @@ async fn create_user( Extension(argon2): Extension>>, Json(nu): Json, ) -> Result<(StatusCode, String)> { - crate::users_ee::create_user(authed, db, webhook, argon2, nu).await + crate::users_oss::create_user(authed, db, webhook, argon2, nu).await } async fn delete_workspace_user( @@ -1582,7 +1582,7 @@ async fn set_password( Json(ep): Json, ) -> Result { let email = authed.email.clone(); - crate::users_ee::set_password(db, argon2, authed, &email, ep).await + crate::users_oss::set_password(db, argon2, authed, &email, ep).await } async fn set_password_of_user( @@ -1593,7 +1593,7 @@ async fn set_password_of_user( Json(ep): Json, ) -> Result { require_super_admin(&db, &authed.email).await?; - crate::users_ee::set_password(db, argon2, authed, &email, ep).await + crate::users_oss::set_password(db, argon2, authed, &email, ep).await } async fn set_login_type( @@ -1750,7 +1750,22 @@ async fn refresh_token( .await? .unwrap_or(false); - let _ = create_session_token(&authed.email, super_admin, &mut tx, cookies).await?; + let new_token = create_session_token(&authed.email, super_admin, &mut tx, cookies).await?; + + audit_log( + &mut *tx, + &AuditAuthor { + email: authed.email.to_string(), + username: authed.email.to_string(), + username_override: None, + }, + "users.token.refresh", + ActionKind::Create, + &"global", + Some(&truncate_token(&new_token)), + None, + ) + .await?; tx.commit().await?; Ok("token refreshed".to_string()) diff --git a/backend/windmill-api/src/users_ee.rs b/backend/windmill-api/src/users_oss.rs similarity index 71% rename from backend/windmill-api/src/users_ee.rs rename to backend/windmill-api/src/users_oss.rs index 7a11239a2f..88a12eb710 100644 --- a/backend/windmill-api/src/users_ee.rs +++ b/backend/windmill-api/src/users_oss.rs @@ -1,15 +1,27 @@ +#[cfg(feature = "private")] +#[allow(unused)] +pub use crate::users_ee::*; + +#[cfg(not(feature = "private"))] use std::sync::Arc; +#[cfg(not(feature = "private"))] use crate::db::ApiAuthed; +#[cfg(not(feature = "private"))] use crate::users::{EditPassword, NewUser}; +#[cfg(not(feature = "private"))] use crate::{db::DB, webhook_util::WebhookShared}; +#[cfg(not(feature = "private"))] use argon2::Argon2; +#[cfg(not(feature = "private"))] use http::StatusCode; +#[cfg(not(feature = "private"))] use windmill_common::error::{Error, Result}; +#[cfg(not(feature = "private"))] pub async fn create_user( _authed: ApiAuthed, _db: DB, @@ -22,6 +34,7 @@ pub async fn create_user( )) } +#[cfg(not(feature = "private"))] pub async fn set_password( _db: DB, _argon2: Arc>, @@ -34,6 +47,7 @@ pub async fn set_password( )) } +#[cfg(not(feature = "private"))] pub fn send_email_if_possible(_subject: &str, _content: &str, _to: &str) { tracing::warn!( "send_email_if_possible is not implemented in Windmill's Open Source repository" diff --git a/backend/windmill-api/src/utils.rs b/backend/windmill-api/src/utils.rs index 5eef389138..8db0d99281 100644 --- a/backend/windmill-api/src/utils.rs +++ b/backend/windmill-api/src/utils.rs @@ -6,8 +6,6 @@ * LICENSE-AGPL for a copy of the license. */ -use std::fmt::Display; - use axum::{body::Body, response::Response}; use regex::Regex; use serde::{Deserialize, Deserializer}; @@ -31,23 +29,6 @@ pub struct WithStarredInfoQuery { pub with_starred_info: Option, } -#[derive(Deserialize)] -#[serde(rename_all = "lowercase")] -pub enum RunnableKind { - Script, - Flow, -} - -impl Display for RunnableKind { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - let runnable_kind = match self { - RunnableKind::Script => "script", - RunnableKind::Flow => "flow" - }; - write!(f, "{}", runnable_kind) - } -} - pub async fn require_super_admin(db: &DB, email: &str) -> error::Result<()> { let is_admin = is_super_admin_email(db, email).await?; diff --git a/backend/windmill-api/src/variables.rs b/backend/windmill-api/src/variables.rs index e810f95750..7dc6a85b49 100644 --- a/backend/windmill-api/src/variables.rs +++ b/backend/windmill-api/src/variables.rs @@ -20,7 +20,7 @@ use axum::{ use hyper::StatusCode; use serde_json::Value; -use windmill_audit::audit_ee::{audit_log, AuditAuthorable}; +use windmill_audit::audit_oss::{audit_log, AuditAuthorable}; use windmill_audit::ActionKind; use windmill_common::{ db::UserDB, @@ -186,7 +186,7 @@ async fn get_variable( #[cfg(feature = "oauth2")] { Some( - crate::oauth2_ee::_refresh_token( + crate::oauth2_oss::_refresh_token( tx, &variable.path, &w_id, @@ -653,7 +653,7 @@ pub async fn get_value_internal<'c>( if variable.is_expired.unwrap_or(false) && variable.account.is_some() { #[cfg(feature = "oauth2")] { - crate::oauth2_ee::_refresh_token( + crate::oauth2_oss::_refresh_token( tx, &variable.path, &w_id, diff --git a/backend/windmill-api/src/websocket_triggers.rs b/backend/windmill-api/src/websocket_triggers.rs index 8bd165844c..8c263b1f62 100644 --- a/backend/windmill-api/src/websocket_triggers.rs +++ b/backend/windmill-api/src/websocket_triggers.rs @@ -20,7 +20,7 @@ use std::{collections::HashMap, fmt}; use tokio::net::TcpStream; use tokio_tungstenite::{connect_async, tungstenite::Message, MaybeTlsStream, WebSocketStream}; use uuid::Uuid; -use windmill_audit::{audit_ee::audit_log, ActionKind}; +use windmill_audit::{audit_oss::audit_log, ActionKind}; use windmill_common::{ db::UserDB, error::{self, to_anyhow, JsonResult}, diff --git a/backend/windmill-api/src/workspaces.rs b/backend/windmill-api/src/workspaces.rs index 75c9a8d034..b8e8a813f3 100644 --- a/backend/windmill-api/src/workspaces.rs +++ b/backend/windmill-api/src/workspaces.rs @@ -10,7 +10,7 @@ use std::collections::HashMap; use crate::ai::{AIConfig, AI_REQUEST_CACHE}; use crate::db::ApiAuthed; -use crate::users_ee::send_email_if_possible; +use crate::users_oss::send_email_if_possible; use crate::utils::get_instance_username_or_create_pending; use crate::BASE_URL; use crate::{ @@ -30,7 +30,7 @@ use chrono::Utc; use regex::Regex; use uuid::Uuid; -use windmill_audit::audit_ee::audit_log; +use windmill_audit::audit_oss::audit_log; use windmill_audit::ActionKind; use windmill_common::db::UserDB; use windmill_common::s3_helpers::LargeFileStorage; @@ -58,7 +58,7 @@ use sqlx::{FromRow, Postgres, Transaction}; use windmill_common::oauth2::InstanceEvent; use windmill_common::utils::not_found_if_none; -use crate::teams_ee::{ +use crate::teams_oss::{ connect_teams, edit_teams_command, run_teams_message_test_job, workspaces_list_available_teams_channels, workspaces_list_available_teams_ids, }; @@ -145,7 +145,7 @@ pub fn workspaced_service() -> Router { #[cfg(all(feature = "stripe", feature = "enterprise"))] { - crate::stripe_ee::add_stripe_routes(router) + crate::stripe_oss::add_stripe_routes(router) } #[cfg(not(feature = "stripe"))] @@ -640,7 +640,7 @@ async fn edit_auto_invite( Path(w_id): Path, Json(ea): Json, ) -> Result { - crate::workspaces_ee::edit_auto_invite(authed, db, w_id, ea).await + crate::workspaces_oss::edit_auto_invite(authed, db, w_id, ea).await } async fn edit_webhook( diff --git a/backend/windmill-api/src/workspaces_export.rs b/backend/windmill-api/src/workspaces_export.rs index cd2be0f708..5673958ab8 100644 --- a/backend/windmill-api/src/workspaces_export.rs +++ b/backend/windmill-api/src/workspaces_export.rs @@ -548,6 +548,8 @@ pub(crate) async fn tarball_workspace( authentication_resource_path, script_path, is_flow, + summary, + description, edited_by, edited_at, email, @@ -622,7 +624,7 @@ pub(crate) async fn tarball_workspace( #[cfg(all(feature = "enterprise", feature = "kafka"))] { let kafka_triggers = sqlx::query_as!( - crate::kafka_triggers_ee::KafkaTrigger, + crate::kafka_triggers_oss::KafkaTrigger, "SELECT * FROM kafka_trigger WHERE workspace_id = $1", &w_id @@ -644,7 +646,7 @@ pub(crate) async fn tarball_workspace( #[cfg(all(feature = "enterprise", feature = "sqs_trigger"))] { let sqs_triggers = sqlx::query_as!( - crate::sqs_triggers_ee::SqsTrigger, + crate::sqs_triggers_oss::SqsTrigger, r#" SELECT aws_auth_resource_type AS "aws_auth_resource_type: _", @@ -684,7 +686,7 @@ pub(crate) async fn tarball_workspace( #[cfg(all(feature = "enterprise", feature = "gcp_trigger"))] { let gcp_triggers = sqlx::query_as!( - crate::gcp_triggers_ee::GcpTrigger, + crate::gcp_triggers_oss::GcpTrigger, r#" SELECT gcp_resource_path, @@ -726,7 +728,7 @@ pub(crate) async fn tarball_workspace( #[cfg(all(feature = "enterprise", feature = "nats"))] { let nats_triggers = sqlx::query_as!( - crate::nats_triggers_ee::NatsTrigger, + crate::nats_triggers_oss::NatsTrigger, "SELECT * FROM nats_trigger WHERE workspace_id = $1", &w_id diff --git a/backend/windmill-api/src/workspaces_extra.rs b/backend/windmill-api/src/workspaces_extra.rs index 07570df5c3..51a224d0e2 100644 --- a/backend/windmill-api/src/workspaces_extra.rs +++ b/backend/windmill-api/src/workspaces_extra.rs @@ -8,7 +8,7 @@ use axum::{ Json, }; -use windmill_audit::audit_ee::audit_log; +use windmill_audit::audit_oss::audit_log; use windmill_audit::ActionKind; use windmill_common::worker::CLOUD_HOSTED; diff --git a/backend/windmill-api/src/workspaces_ee.rs b/backend/windmill-api/src/workspaces_oss.rs similarity index 70% rename from backend/windmill-api/src/workspaces_ee.rs rename to backend/windmill-api/src/workspaces_oss.rs index aa8799e233..4f55539da1 100644 --- a/backend/windmill-api/src/workspaces_ee.rs +++ b/backend/windmill-api/src/workspaces_oss.rs @@ -1,8 +1,14 @@ +#[cfg(feature = "private")] +#[allow(unused)] +pub use crate::workspaces_ee::*; + +#[cfg(not(feature = "private"))] use crate::{ db::{ApiAuthed, DB}, workspaces::EditAutoInvite, }; +#[cfg(not(feature = "private"))] pub async fn edit_auto_invite( _authed: ApiAuthed, _db: DB, diff --git a/backend/windmill-audit/Cargo.toml b/backend/windmill-audit/Cargo.toml index 8b202b4abf..5a9ea376b6 100644 --- a/backend/windmill-audit/Cargo.toml +++ b/backend/windmill-audit/Cargo.toml @@ -10,6 +10,7 @@ path = "./src/lib.rs" [features] enterprise = ["windmill-common/enterprise"] +private = [] [dependencies] serde.workspace = true diff --git a/backend/windmill-audit/src/audit_ee.rs b/backend/windmill-audit/src/audit_oss.rs similarity index 78% rename from backend/windmill-audit/src/audit_ee.rs rename to backend/windmill-audit/src/audit_oss.rs index 97a8e6bbcf..d29daaa7ae 100644 --- a/backend/windmill-audit/src/audit_ee.rs +++ b/backend/windmill-audit/src/audit_oss.rs @@ -1,3 +1,7 @@ +#[cfg(feature = "private")] +#[allow(unused)] +pub use crate::audit_ee::*; + /* * Author: Ruben Fiszel * Copyright: Windmill Labs, Inc 2022 @@ -5,23 +9,26 @@ * Please see the included NOTICE for copyright information and * LICENSE-AGPL for a copy of the license. */ -use std::collections::HashMap; - -use windmill_common::{ - error::{Error, Result}, - utils::Pagination, +#[cfg(not(feature = "private"))] +use { + crate::{ActionKind, AuditLog, ListAuditLogQuery}, + sqlx::{Postgres, Transaction}, + std::collections::HashMap, + windmill_common::{ + error::{Error, Result}, + utils::Pagination, + }, }; -use crate::{ActionKind, AuditLog, ListAuditLogQuery}; -use sqlx::{Postgres, Transaction}; - #[derive(Clone)] +#[cfg(not(feature = "private"))] pub struct AuditAuthor { pub username: String, pub email: String, pub username_override: Option, } +#[cfg(not(feature = "private"))] impl AuditAuthorable for AuditAuthor { fn email(&self) -> &str { &self.email @@ -36,12 +43,14 @@ impl AuditAuthorable for AuditAuthor { } } +#[cfg(not(feature = "private"))] pub trait AuditAuthorable { fn username(&self) -> &str; fn email(&self) -> &str; fn username_override(&self) -> Option<&str>; } +#[cfg(not(feature = "private"))] #[tracing::instrument(level = "trace", skip_all)] pub async fn audit_log<'c, E: sqlx::Executor<'c, Database = Postgres>>( _db: E, @@ -56,6 +65,7 @@ pub async fn audit_log<'c, E: sqlx::Executor<'c, Database = Postgres>>( Ok(()) } +#[cfg(not(feature = "private"))] pub async fn list_audit( _tx: Transaction<'_, Postgres>, _w_id: String, @@ -66,6 +76,7 @@ pub async fn list_audit( return Ok(vec![]); } +#[cfg(not(feature = "private"))] pub async fn get_audit(tx: Transaction<'_, Postgres>, _id: i32, _w_id: &str) -> Result { // Implementation is not open source as Audit logs is a Windmill Enterprise Edition feature tx.commit().await?; diff --git a/backend/windmill-audit/src/lib.rs b/backend/windmill-audit/src/lib.rs index 10894798fb..15bda522c1 100644 --- a/backend/windmill-audit/src/lib.rs +++ b/backend/windmill-audit/src/lib.rs @@ -1,7 +1,9 @@ use serde::{Deserialize, Serialize}; use sqlx::FromRow; +#[cfg(feature = "private")] pub mod audit_ee; +pub mod audit_oss; #[derive(sqlx::Type, Serialize, Deserialize, Debug)] #[sqlx(type_name = "ACTION_KIND", rename_all = "lowercase")] diff --git a/backend/windmill-autoscaling/Cargo.toml b/backend/windmill-autoscaling/Cargo.toml index fbebaf0fd7..7e4bc623ad 100644 --- a/backend/windmill-autoscaling/Cargo.toml +++ b/backend/windmill-autoscaling/Cargo.toml @@ -10,6 +10,7 @@ path = "./src/lib.rs" [features] enterprise = ["windmill-queue/enterprise", "windmill-common/enterprise"] +private = [] default = [] [dependencies] diff --git a/backend/windmill-autoscaling/src/autoscaling_ee.rs b/backend/windmill-autoscaling/src/autoscaling_ee.rs deleted file mode 100644 index 1c9defbede..0000000000 --- a/backend/windmill-autoscaling/src/autoscaling_ee.rs +++ /dev/null @@ -1,6 +0,0 @@ -use windmill_common::DB; - -pub async fn apply_all_autoscaling(_db: &DB) -> anyhow::Result<()> { - // Autoscaling is an ee feature - Ok(()) -} diff --git a/backend/windmill-autoscaling/src/autoscaling_oss.rs b/backend/windmill-autoscaling/src/autoscaling_oss.rs new file mode 100644 index 0000000000..c9cbdc3ee0 --- /dev/null +++ b/backend/windmill-autoscaling/src/autoscaling_oss.rs @@ -0,0 +1,12 @@ +#[cfg(feature = "private")] +#[allow(unused)] +pub use crate::autoscaling_ee::*; + +#[cfg(not(feature = "private"))] +use windmill_common::DB; + +#[cfg(not(feature = "private"))] +pub async fn apply_all_autoscaling(_db: &DB) -> anyhow::Result<()> { + // Autoscaling is an ee feature + Ok(()) +} diff --git a/backend/windmill-autoscaling/src/lib.rs b/backend/windmill-autoscaling/src/lib.rs index 28b9319244..1b9d6b94bf 100644 --- a/backend/windmill-autoscaling/src/lib.rs +++ b/backend/windmill-autoscaling/src/lib.rs @@ -1,2 +1,4 @@ -mod autoscaling_ee; -pub use autoscaling_ee::*; +#[cfg(feature = "private")] +pub mod autoscaling_ee; +mod autoscaling_oss; +pub use autoscaling_oss::*; diff --git a/backend/windmill-common/Cargo.toml b/backend/windmill-common/Cargo.toml index 17764b1253..250f48e336 100644 --- a/backend/windmill-common/Cargo.toml +++ b/backend/windmill-common/Cargo.toml @@ -7,6 +7,7 @@ edition.workspace = true [features] default = [] enterprise = [] +private = [] jemalloc = ["dep:tikv-jemalloc-ctl"] tantivy = [] prometheus = ["dep:prometheus"] @@ -79,6 +80,7 @@ backon.workspace = true openidconnect = { workspace = true, optional = true } strum.workspace = true strum_macros.workspace = true +url.workspace = true semver.workspace = true croner = "2.0.6" diff --git a/backend/windmill-common/src/agent_workers.rs b/backend/windmill-common/src/agent_workers.rs index 7c91ec2165..52c7e2d70c 100644 --- a/backend/windmill-common/src/agent_workers.rs +++ b/backend/windmill-common/src/agent_workers.rs @@ -36,6 +36,7 @@ pub struct AgentAuth { } pub const AGENT_JWT_PREFIX: &str = "jwt_agent_"; + pub fn build_agent_http_client(worker_suffix: &str) -> HttpClient { let client = ClientBuilder::new( reqwest::Client::builder() diff --git a/backend/windmill-common/src/client.rs b/backend/windmill-common/src/client.rs index 0c5394c6e6..6bc1d12ef9 100644 --- a/backend/windmill-common/src/client.rs +++ b/backend/windmill-common/src/client.rs @@ -17,6 +17,15 @@ pub struct AuthedClient { } impl AuthedClient { + pub fn new( + base_internal_url: String, + workspace: String, + token: String, + force_client: Option, + ) -> AuthedClient { + AuthedClient { base_internal_url, workspace, token, force_client } + } + pub async fn get(&self, url: &str, query: Vec<(&str, String)>) -> anyhow::Result { self.force_client .as_ref() diff --git a/backend/windmill-common/src/ee.rs b/backend/windmill-common/src/ee_oss.rs similarity index 63% rename from backend/windmill-common/src/ee.rs rename to backend/windmill-common/src/ee_oss.rs index 7acb430f1b..51b1efd2e2 100644 --- a/backend/windmill-common/src/ee.rs +++ b/backend/windmill-common/src/ee_oss.rs @@ -1,24 +1,35 @@ -#[cfg(feature = "enterprise")] +#[cfg(feature = "private")] +#[allow(unused)] +pub use crate::ee::*; + +#[cfg(all(feature = "enterprise", not(feature = "private")))] use crate::db::DB; -use crate::ee::LicensePlan::Community; -#[cfg(feature = "enterprise")] +#[cfg(not(feature = "private"))] +use crate::ee_oss::LicensePlan::Community; +#[cfg(all(feature = "enterprise", not(feature = "private")))] use crate::error; +#[cfg(not(feature = "private"))] use serde::Deserialize; +#[cfg(not(feature = "private"))] use std::sync::Arc; +#[cfg(not(feature = "private"))] use tokio::sync::RwLock; +#[cfg(not(feature = "private"))] lazy_static::lazy_static! { pub static ref LICENSE_KEY_VALID: Arc> = Arc::new(RwLock::new(true)); pub static ref LICENSE_KEY_ID: Arc> = Arc::new(RwLock::new("".to_string())); pub static ref LICENSE_KEY: Arc> = Arc::new(RwLock::new("".to_string())); } +#[cfg(not(feature = "private"))] pub enum LicensePlan { Community, Pro, Enterprise, } +#[cfg(not(feature = "private"))] pub async fn get_license_plan() -> LicensePlan { // Implementation is not open source return Community; @@ -26,6 +37,7 @@ pub async fn get_license_plan() -> LicensePlan { #[derive(Deserialize)] #[serde(untagged)] +#[cfg(not(feature = "private"))] pub enum CriticalErrorChannel { Email { email: String }, Slack { slack_channel: String }, @@ -33,6 +45,7 @@ pub enum CriticalErrorChannel { } #[derive(Deserialize)] +#[cfg(not(feature = "private"))] pub struct TeamsChannel { pub team_id: String, pub team_name: String, @@ -40,6 +53,7 @@ pub struct TeamsChannel { pub channel_name: String, } +#[cfg(not(feature = "private"))] pub enum CriticalAlertKind { #[cfg(feature = "enterprise")] CriticalError, @@ -47,7 +61,7 @@ pub enum CriticalAlertKind { RecoveredCriticalError, } -#[cfg(feature = "enterprise")] +#[cfg(all(feature = "enterprise", not(feature = "private")))] pub async fn send_critical_alert( _error_message: String, _db: &DB, @@ -56,7 +70,7 @@ pub async fn send_critical_alert( ) { } -#[cfg(feature = "enterprise")] +#[cfg(all(feature = "enterprise", not(feature = "private")))] pub async fn maybe_renew_license_key_on_start( _http_client: &reqwest::Client, _db: &crate::db::DB, @@ -66,14 +80,14 @@ pub async fn maybe_renew_license_key_on_start( force_renew_now } -#[cfg(feature = "enterprise")] +#[cfg(all(feature = "enterprise", not(feature = "private")))] pub enum RenewReason { Manual, Schedule, OnStart, } -#[cfg(feature = "enterprise")] +#[cfg(all(feature = "enterprise", not(feature = "private")))] pub async fn renew_license_key( _http_client: &reqwest::Client, _db: &crate::db::DB, @@ -84,7 +98,7 @@ pub async fn renew_license_key( "".to_string() } -#[cfg(feature = "enterprise")] +#[cfg(all(feature = "enterprise", not(feature = "private")))] pub async fn create_customer_portal_session( _http_client: &reqwest::Client, _key: Option, @@ -93,14 +107,18 @@ pub async fn create_customer_portal_session( Ok("".to_string()) } -#[cfg(feature = "enterprise")] +#[cfg(all(feature = "enterprise", not(feature = "private")))] pub async fn worker_groups_alerts(_db: &DB) {} -#[cfg(feature = "enterprise")] +#[cfg(all(feature = "enterprise", not(feature = "private")))] pub async fn jobs_waiting_alerts(_db: &DB) {} -#[cfg(feature = "enterprise")] -pub async fn low_disk_alerts(_db: &DB, _server_mode: bool, _worker_mode: bool, _workers: Vec) { +#[cfg(all(feature = "enterprise", not(feature = "private")))] +pub async fn low_disk_alerts( + _db: &DB, + _server_mode: bool, + _worker_mode: bool, + _workers: Vec, +) { // Implementation is not open source } - diff --git a/backend/windmill-common/src/email_ee.rs b/backend/windmill-common/src/email_oss.rs similarity index 61% rename from backend/windmill-common/src/email_ee.rs rename to backend/windmill-common/src/email_oss.rs index 42aebbeec3..e6a340523e 100644 --- a/backend/windmill-common/src/email_ee.rs +++ b/backend/windmill-common/src/email_oss.rs @@ -1,5 +1,11 @@ +#[cfg(feature = "private")] +#[allow(unused)] +pub use crate::email_ee::*; + +#[cfg(not(feature = "private"))] use crate::server::Smtp; +#[cfg(not(feature = "private"))] pub async fn send_email( _subject: &str, _content: &str, diff --git a/backend/windmill-common/src/job_s3_helpers_ee.rs b/backend/windmill-common/src/job_s3_helpers_oss.rs similarity index 76% rename from backend/windmill-common/src/job_s3_helpers_ee.rs rename to backend/windmill-common/src/job_s3_helpers_oss.rs index d5d6ac4052..b7ebd57f3e 100644 --- a/backend/windmill-common/src/job_s3_helpers_ee.rs +++ b/backend/windmill-common/src/job_s3_helpers_oss.rs @@ -1,5 +1,11 @@ +#[cfg(feature = "private")] +#[allow(unused)] +pub use crate::job_s3_helpers_ee::*; + +#[cfg(not(feature = "private"))] use crate::s3_helpers::{ObjectStoreResource, StorageResourceType}; +#[cfg(not(feature = "private"))] pub async fn get_s3_resource_internal<'c>( _resource_type: StorageResourceType, _s3_resource_value_raw: serde_json::Value, @@ -9,11 +15,13 @@ pub async fn get_s3_resource_internal<'c>( todo!() } +#[cfg(not(feature = "private"))] pub enum TokenGenerator<'c> { AsClient(&'c crate::client::AuthedClient), AsServerInstance(), } +#[cfg(not(feature = "private"))] impl<'c> TokenGenerator<'c> { pub async fn gen_token( &self, @@ -24,7 +32,7 @@ impl<'c> TokenGenerator<'c> { } } -#[cfg(feature = "parquet")] +#[cfg(all(feature = "parquet", not(feature = "private")))] pub(crate) async fn generate_s3_aws_oidc_resource<'c>( _clone: crate::s3_helpers::S3AwsOidcResource, _token_generator: TokenGenerator<'c>, diff --git a/backend/windmill-common/src/lib.rs b/backend/windmill-common/src/lib.rs index 6d92cd8f55..498beecd6c 100644 --- a/backend/windmill-common/src/lib.rs +++ b/backend/windmill-common/src/lib.rs @@ -19,7 +19,7 @@ use std::{ use tokio::sync::broadcast; -use ee::CriticalErrorChannel; +use ee_oss::CriticalErrorChannel; use error::Error; use scripts::ScriptLang; use sqlx::{Pool, Postgres}; @@ -32,8 +32,12 @@ pub mod bench; pub mod cache; pub mod client; pub mod db; +#[cfg(feature = "private")] pub mod ee; +pub mod ee_oss; +#[cfg(feature = "private")] pub mod email_ee; +pub mod email_oss; pub mod error; pub mod external_ip; pub mod flow_status; @@ -41,25 +45,35 @@ pub mod flows; pub mod global_settings; pub mod indexer; pub mod job_metrics; -#[cfg(feature = "parquet")] +#[cfg(all(feature = "parquet", feature = "private"))] pub mod job_s3_helpers_ee; +#[cfg(feature = "parquet")] +pub mod job_s3_helpers_oss; -#[cfg(all(feature = "enterprise", feature = "openidconnect"))] +#[cfg(all(feature = "enterprise", feature = "openidconnect", feature = "private"))] pub mod oidc_ee; +#[cfg(all(feature = "enterprise", feature = "openidconnect"))] +pub mod oidc_oss; pub mod jobs; pub mod jwt; pub mod more_serde; pub mod oauth2; +#[cfg(feature = "private")] pub mod otel_ee; +pub mod otel_oss; pub mod queue; pub mod s3_helpers; pub mod schedule; pub mod schema; pub mod scripts; pub mod server; +#[cfg(feature = "private")] pub mod stats_ee; +pub mod stats_oss; +#[cfg(feature = "private")] pub mod teams_ee; +pub mod teams_oss; pub mod tracing_init; pub mod users; pub mod utils; @@ -135,6 +149,8 @@ lazy_static::lazy_static! { pub static ref DEPLOYED_SCRIPT_INFO_CACHE: Cache<(String, i64), ScriptHashInfo> = Cache::new(1000); pub static ref FLOW_INFO_CACHE: Cache<(String, i64), FlowVersionInfo> = Cache::new(1000); + pub static ref QUIET_LOGS: bool = std::env::var("QUIET_LOGS").map(|s| s.parse::().unwrap_or(false)).unwrap_or(false); + } const LATEST_VERSION_ID_CACHE_TTL: std::time::Duration = std::time::Duration::from_secs(60); diff --git a/backend/windmill-common/src/oidc_ee.rs b/backend/windmill-common/src/oidc_ee.rs deleted file mode 100644 index e7a157b04d..0000000000 --- a/backend/windmill-common/src/oidc_ee.rs +++ /dev/null @@ -1,198 +0,0 @@ -/* - * Author: Ruben Fiszel - * Copyright: Windmill Labs, Inc 2023 - * This file and its contents are licensed under the AGPLv3 License. - * Please see the included NOTICE for copyright information and - * LICENSE-AGPL for a copy of the license. - */ - -use serde::{Deserialize, Serialize}; -use tokio::sync::RwLock; -#[cfg(all(feature = "enterprise", feature = "openidconnect"))] -use { - crate::db::DB, - crate::{auth::IdToken as WindmillIdToken, error::Result}, - anyhow, - openidconnect::{ - core::{CoreJwsSigningAlgorithm, CoreRsaPrivateSigningKey}, - IssuerUrl, JsonWebKeyId, - }, - std::process::Command, -}; - -#[cfg(feature = "openidconnect")] -use openidconnect::AdditionalClaims; - -#[cfg(feature = "openidconnect")] -impl AdditionalClaims for JobClaim {} - -#[cfg(feature = "openidconnect")] -impl AdditionalClaims for WorkspaceClaim {} - -#[cfg(feature = "openidconnect")] -impl AdditionalClaims for InstanceClaim {} - -#[derive(Clone, Debug, Default, Deserialize, PartialEq, Eq, Serialize)] -pub struct WorkspaceClaim { - pub workspace: String, -} - -#[derive(Clone, Debug, Default, Deserialize, PartialEq, Eq, Serialize)] -pub struct InstanceClaim {} - -#[derive(Clone, Debug, Default, Deserialize, PartialEq, Eq, Serialize)] -pub struct JobClaim { - pub job_id: String, - pub path: Option, - pub flow_path: Option, - pub groups: Vec, - pub username: String, - pub email: String, - pub workspace: String, -} - -lazy_static::lazy_static! { - static ref PRIVATE_KEY: RwLock> = RwLock::new(None); -} - -pub async fn generate_id_token( - db: Option<&DB>, - claim: T, - audience: &str, - identifier: String, - email: Option, -) -> Result { - use chrono::{Duration, Utc}; - use openidconnect::{ - core::{CoreGenderClaim, CoreJweContentEncryptionAlgorithm}, - Audience, EndUserEmail, IdToken, IdTokenClaims, StandardClaims, SubjectIdentifier, - }; - - let private_key = get_private_key(db).await?; - - let issue_url = format!("{}/api/oidc/", crate::BASE_URL.read().await.clone()); - let issue_time = Utc::now(); - let expiration = issue_time + Duration::try_hours(48).unwrap(); - let id_token = IdToken::< - T, - CoreGenderClaim, - CoreJweContentEncryptionAlgorithm, - CoreJwsSigningAlgorithm, - >::new( - IdTokenClaims::::new( - // Specify the issuer URL for the OpenID Connect Provider. - IssuerUrl::new(issue_url) - .map_err(|e| anyhow::anyhow!("Failed to generate IssueUrl: {}", e))?, - // The audience is usually a single entry with the client ID of the client for whom - // the ID token is intended. This is a required claim. - vec![Audience::new(audience.to_string())], - // The ID token expiration is usually much shorter than that of the access or refresh - // tokens issued to clients. - expiration, - // The issue time is usually the current time. - issue_time, - // Set the standard claims defined by the OpenID Connect Core spec. - StandardClaims::new( - // Stable subject identifiers are recommended in place of e-mail addresses or other - // potentially unstable identifiers. This is the only required claim. - SubjectIdentifier::new(identifier), - ) - // Optional: specify the user's e-mail address. This should only be provided if the - // client has been granted the 'profile' or 'email' scopes. - .set_email(email.map(|x| EndUserEmail::new(x))) - // Optional: specify whether the provider has verified the user's e-mail address. - .set_email_verified(Some(true)), - // OpenID Connect Providers may supply custom claims by providing a struct that - // implements the AdditionalClaims trait. This requires manually using the - // generic IdTokenClaims struct rather than the CoreIdTokenClaims type alias, - // however. - claim, - ), - // The private key used for signing the ID token. For confidential clients (those able - // to maintain a client secret), a CoreHmacKey can also be used, in conjunction - // with one of the CoreJwsSigningAlgorithm::HmacSha* signing algorithms. When using an - // HMAC-based signing algorithm, the UTF-8 representation of the client secret should - // be used as the HMAC key. - &CoreRsaPrivateSigningKey::from_pem( - &private_key, - Some(JsonWebKeyId::new("windmill".to_string())), - ) - .map_err(|e| anyhow::anyhow!("Invalid private key: {}", e))?, - // Uses the RS256 signature algorithm. This crate supports any RS*, PS*, or HS* - // signature algorithm. - CoreJwsSigningAlgorithm::RsaSsaPkcs1V15Sha256, - // When returning the ID token alongside an access token (e.g., in the Authorization Code - // flow), it is recommended to pass the access token here to set the `at_hash` claim - // automatically. - None, - // When returning the ID token alongside an authorization code (e.g., in the implicit - // flow), it is recommended to pass the authorization code here to set the `c_hash` claim - // automatically. - None, - ) - .map_err(|e| anyhow::anyhow!("Failed to generate token: {}", e))?; - - Ok(WindmillIdToken::new(id_token.to_string(), expiration)) -} - -#[cfg(all(feature = "enterprise", feature = "openidconnect"))] -pub async fn get_private_key(db: Option<&DB>) -> anyhow::Result { - if let Some(key) = PRIVATE_KEY.read().await.clone() { - return Ok(key); - } else if let Some(db) = db { - let key = sqlx::query_scalar!( - "SELECT value->>'private_key' FROM global_settings WHERE name = 'rsa_keys'", - ) - .fetch_optional(db) - .await? - .flatten(); - - let key = key.filter(|s| !s.is_empty()); - - if let Some(key) = key { - return Ok(key); - } else { - let keys = gen_pems(db).await?; - return Ok(keys.private_key); - } - } else { - return Err(anyhow::anyhow!("Private key not found and no db provided")); - } -} - -#[cfg(all(feature = "enterprise", feature = "openidconnect"))] -#[derive(Debug, Clone, serde::Serialize)] -struct Keys { - private_key: String, -} - -#[cfg(all(feature = "enterprise", feature = "openidconnect"))] -async fn gen_pems(db: &DB) -> anyhow::Result { - use anyhow::anyhow; - - let private_key_cmd = Command::new("openssl") - .arg("genrsa") - .arg("--traditional") - .arg("2048") - .output() - .expect("failed to execute process"); - - let private_key = String::from_utf8(private_key_cmd.stdout)?; - - tracing::debug!("Generated private key: {}", private_key); - - if private_key.is_empty() { - return Err(anyhow!("Failed to generate RSA key: key is empty")); - } - - let keys = Keys { private_key }; - - sqlx::query!( - r#"INSERT INTO global_settings (name, value) VALUES ('rsa_keys', $1)"#, - serde_json::to_value(&keys).unwrap() - ) - .execute(db) - .await?; - - Ok(keys) -} diff --git a/backend/windmill-common/src/oidc_oss.rs b/backend/windmill-common/src/oidc_oss.rs new file mode 100644 index 0000000000..0a42cee2ed --- /dev/null +++ b/backend/windmill-common/src/oidc_oss.rs @@ -0,0 +1,92 @@ +#[cfg(feature = "private")] +#[allow(unused)] +pub use crate::oidc_ee::*; + +/* + * Author: Ruben Fiszel + * Copyright: Windmill Labs, Inc 2023 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + +#[cfg(not(feature = "private"))] +use serde::{Deserialize, Serialize}; +#[cfg(not(feature = "private"))] +use tokio::sync::RwLock; +#[cfg(all( + feature = "enterprise", + feature = "openidconnect", + not(feature = "private") +))] +use { + crate::db::DB, + crate::{ + auth::IdToken as WindmillIdToken, + error::{Error, Result}, + }, + anyhow, +}; + +#[cfg(all(feature = "openidconnect", not(feature = "private")))] +use openidconnect::AdditionalClaims; + +#[cfg(all(feature = "openidconnect", not(feature = "private")))] +impl AdditionalClaims for JobClaim {} + +#[cfg(all(feature = "openidconnect", not(feature = "private")))] +impl AdditionalClaims for WorkspaceClaim {} + +#[cfg(all(feature = "openidconnect", not(feature = "private")))] +impl AdditionalClaims for InstanceClaim {} + +#[derive(Clone, Debug, Default, Deserialize, PartialEq, Eq, Serialize)] +#[cfg(not(feature = "private"))] +pub struct WorkspaceClaim { + pub workspace: String, +} + +#[derive(Clone, Debug, Default, Deserialize, PartialEq, Eq, Serialize)] +#[cfg(not(feature = "private"))] +pub struct InstanceClaim {} + +#[derive(Clone, Debug, Default, Deserialize, PartialEq, Eq, Serialize)] +#[cfg(not(feature = "private"))] +pub struct JobClaim { + pub job_id: String, + pub path: Option, + pub flow_path: Option, + pub groups: Vec, + pub username: String, + pub email: String, + pub workspace: String, +} + +#[cfg(not(feature = "private"))] +lazy_static::lazy_static! { + static ref PRIVATE_KEY: RwLock> = RwLock::new(None); +} + +#[cfg(not(feature = "private"))] +pub async fn generate_id_token( + _db: Option<&DB>, + _claim: T, + _audience: &str, + _identifier: String, + _email: Option, +) -> Result { + Err(Error::internal_err( + "Not implemented in Windmill's Open Source repository".to_string(), + )) +} + +#[cfg(all( + feature = "enterprise", + feature = "openidconnect", + not(feature = "private") +))] +pub async fn get_private_key(_db: Option<&DB>) -> anyhow::Result { + Err(anyhow::anyhow!( + "Not implemented in Windmill's Open Source repository" + )) +} diff --git a/backend/windmill-common/src/otel_ee.rs b/backend/windmill-common/src/otel_oss.rs similarity index 60% rename from backend/windmill-common/src/otel_ee.rs rename to backend/windmill-common/src/otel_oss.rs index f3ada162f6..9225915636 100644 --- a/backend/windmill-common/src/otel_ee.rs +++ b/backend/windmill-common/src/otel_oss.rs @@ -1,3 +1,7 @@ +#[cfg(feature = "private")] +#[allow(unused)] +pub use crate::otel_ee::*; + /* * Author: Ruben Fiszel * Copyright: Windmill Labs, Inc 2022 @@ -6,43 +10,51 @@ * LICENSE-AGPL for a copy of the license. */ +#[cfg(not(feature = "private"))] use crate::{jobs::QueuedJob, utils::Mode}; +#[cfg(not(feature = "private"))] use uuid::Uuid; +#[cfg(not(feature = "private"))] pub fn set_span_parent(_span: &tracing::Span, _rj: &Uuid) {} -#[cfg(not(all(feature = "otel", feature = "enterprise")))] +#[cfg(all( + not(all(feature = "otel", feature = "enterprise")), + not(feature = "private") +))] pub(crate) type OtelProvider = Option<()>; -#[cfg(all(feature = "otel", feature = "enterprise"))] +#[cfg(all(feature = "otel", feature = "enterprise", not(feature = "private")))] pub(crate) type OtelProvider = Option; -#[cfg(not(feature = "otel"))] +#[cfg(all(not(feature = "otel"), not(feature = "private")))] pub fn otel_ctx() -> () {} -#[cfg(feature = "otel")] +#[cfg(all(feature = "otel", not(feature = "private")))] #[inline(always)] pub fn otel_ctx() -> opentelemetry::Context { opentelemetry::Context::current() } -#[cfg(not(feature = "otel"))] +#[cfg(all(not(feature = "otel"), not(feature = "private")))] impl FutureExt for T {} -#[cfg(not(feature = "otel"))] +#[cfg(all(not(feature = "otel"), not(feature = "private")))] pub trait FutureExt: Sized { fn with_context(self, _otel_cx: ()) -> Self { self } } +#[cfg(not(feature = "private"))] use tracing_subscriber::EnvFilter; +#[cfg(not(feature = "private"))] pub(crate) fn init_logs_bridge(_mode: &Mode, _hostname: &str, _env: &str) -> Option { None } -#[cfg(all(feature = "otel", feature = "enterprise"))] +#[cfg(all(feature = "otel", feature = "enterprise", not(feature = "private")))] pub(crate) fn init_otlp_tracer( _mode: &Mode, _hostname: &str, @@ -51,8 +63,10 @@ pub(crate) fn init_otlp_tracer( None } +#[cfg(not(feature = "private"))] pub(crate) fn init_meter_provider(_mode: &Mode, _hostname: &str, _env: &str) -> OtelProvider { None } +#[cfg(not(feature = "private"))] pub fn add_root_flow_job_to_otlp(_queued_job: &QueuedJob, _success: bool) {} diff --git a/backend/windmill-common/src/s3_helpers.rs b/backend/windmill-common/src/s3_helpers.rs index 7e7c48851e..171958f5cd 100644 --- a/backend/windmill-common/src/s3_helpers.rs +++ b/backend/windmill-common/src/s3_helpers.rs @@ -147,7 +147,7 @@ pub enum ObjectStoreReload { #[cfg(feature = "parquet")] pub async fn reload_object_store_setting(db: &crate::DB) -> ObjectStoreReload { use crate::{ - ee::{get_license_plan, LicensePlan}, + ee_oss::{get_license_plan, LicensePlan}, global_settings::{load_value_from_global_settings, OBJECT_STORE_CONFIG_SETTING}, s3_helpers::ObjectSettings, }; @@ -613,8 +613,8 @@ pub async fn build_object_store_from_settings( build_azure_blob_client(&azure_blob_resource).map(|x| ExpirableObjectStore::from(x)) } ObjectSettings::AwsOidc(ref s3_aws_oidc_settings) => { - let token_generator = crate::job_s3_helpers_ee::TokenGenerator::AsServerInstance(); - let res = crate::job_s3_helpers_ee::generate_s3_aws_oidc_resource( + let token_generator = crate::job_s3_helpers_oss::TokenGenerator::AsServerInstance(); + let res = crate::job_s3_helpers_oss::generate_s3_aws_oidc_resource( s3_aws_oidc_settings.clone(), token_generator, init_private_key, diff --git a/backend/windmill-common/src/scripts.rs b/backend/windmill-common/src/scripts.rs index dda9daf0f4..ffa7bd486a 100644 --- a/backend/windmill-common/src/scripts.rs +++ b/backend/windmill-common/src/scripts.rs @@ -9,6 +9,7 @@ use std::{ fmt::{self, Display}, hash::{Hash, Hasher}, + str::FromStr, }; use crate::{ @@ -21,6 +22,7 @@ use crate::worker::HUB_CACHE_DIR; use anyhow::Context; use backon::ConstantBuilder; use backon::{BackoffBuilder, Retryable}; +use itertools::Itertools; use serde::de::Error as _; use serde::{ser::SerializeSeq, Deserialize, Deserializer, Serialize}; @@ -85,6 +87,40 @@ impl ScriptLang { } } +impl FromStr for ScriptLang { + type Err = Error; + fn from_str(s: &str) -> Result { + let language = match s.to_lowercase().as_str() { + "bun" => ScriptLang::Bun, + "bunnative" => ScriptLang::Bunnative, + "nativets" => ScriptLang::Nativets, + "deno" => ScriptLang::Deno, + "python3" => ScriptLang::Python3, + "go" => ScriptLang::Go, + "bash" => ScriptLang::Bash, + "powershell" => ScriptLang::Powershell, + "postgresql" => ScriptLang::Postgresql, + "mysql" => ScriptLang::Mysql, + "bigquery" => ScriptLang::Bigquery, + "snowflake" => ScriptLang::Snowflake, + "mssql" => ScriptLang::Mssql, + "graphql" => ScriptLang::Graphql, + "oracledb" => ScriptLang::OracleDB, + "php" => ScriptLang::Php, + "rust" => ScriptLang::Rust, + "ansible" => ScriptLang::Ansible, + "csharp" => ScriptLang::CSharp, + "nu" => ScriptLang::Nu, + "java" => ScriptLang::Java, + language => { + return Err(anyhow::anyhow!("{} is currently not supported", language).into()) + } + }; + + Ok(language) + } +} + #[derive(Eq, PartialEq, Debug, Hash, Clone, Copy, sqlx::Type)] #[sqlx(transparent)] pub struct ScriptHash(pub i64); @@ -367,7 +403,7 @@ where deserializer.deserialize_any(StringOrArrayVisitor) } -#[derive(Deserialize)] +#[derive(Debug, Deserialize)] pub struct ListScriptQuery { pub path_start: Option, pub path_exact: Option, @@ -384,6 +420,29 @@ pub struct ListScriptQuery { pub include_without_main: Option, pub include_draft_only: Option, pub with_deployment_msg: Option, + #[serde(default, deserialize_with = "from_seq")] + pub languages: Option>, +} + +fn from_seq<'de, D>(deserializer: D) -> Result>, D::Error> +where + D: Deserializer<'de>, +{ + let s = ::deserialize(deserializer)?; + + let languages: Vec = s + .split(",") + .map(ScriptLang::from_str) + .try_collect() + .map_err(|e| serde::de::Error::custom(e.to_string()))?; + + let languages = if languages.is_empty() { + None + } else { + Some(languages) + }; + + Ok(languages) } pub fn to_i64(s: &str) -> crate::error::Result { diff --git a/backend/windmill-common/src/stats_ee.rs b/backend/windmill-common/src/stats_oss.rs similarity index 73% rename from backend/windmill-common/src/stats_ee.rs rename to backend/windmill-common/src/stats_oss.rs index 5d2dc82b82..829e6e3a5b 100644 --- a/backend/windmill-common/src/stats_ee.rs +++ b/backend/windmill-common/src/stats_oss.rs @@ -1,17 +1,26 @@ +#[cfg(feature = "private")] +#[allow(unused)] +pub use crate::stats_ee::*; + +#[cfg(not(feature = "private"))] use sqlx::Postgres; +#[cfg(not(feature = "private"))] use crate::{error::Result, scripts::ScriptLang, DB}; +#[cfg(not(feature = "private"))] pub async fn get_disable_stats_setting(_db: &DB) -> bool { // stats details are closed source false } +#[cfg(not(feature = "private"))] pub async fn schedule_stats(_db: &DB, _http_client: &reqwest::Client) -> () { // stats details are closed source } +#[cfg(not(feature = "private"))] #[derive(Debug, sqlx::FromRow, serde::Serialize)] struct JobsUsage { language: Option, @@ -19,12 +28,14 @@ struct JobsUsage { count: i64, } +#[cfg(not(feature = "private"))] pub enum SendStatsReason { Manual, Schedule, OnStart, } +#[cfg(not(feature = "private"))] pub async fn send_stats( _http_client: &reqwest::Client, _db: &DB, @@ -34,11 +45,13 @@ pub async fn send_stats( Ok(()) } +#[cfg(not(feature = "private"))] pub struct ActiveUserUsage { pub author_count: Option, pub operator_count: Option, } +#[cfg(not(feature = "private"))] pub async fn get_user_usage<'c, E: sqlx::Executor<'c, Database = Postgres>>( _db: E, ) -> Result { diff --git a/backend/windmill-common/src/teams_ee.rs b/backend/windmill-common/src/teams_ee.rs deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/backend/windmill-common/src/teams_oss.rs b/backend/windmill-common/src/teams_oss.rs new file mode 100644 index 0000000000..347ad8e6d7 --- /dev/null +++ b/backend/windmill-common/src/teams_oss.rs @@ -0,0 +1,5 @@ +#[cfg(feature = "private")] +#[allow(unused)] +pub use crate::teams_ee::*; + + diff --git a/backend/windmill-common/src/tracing_init.rs b/backend/windmill-common/src/tracing_init.rs index 5ed5cec752..77194ac0a9 100644 --- a/backend/windmill-common/src/tracing_init.rs +++ b/backend/windmill-common/src/tracing_init.rs @@ -48,7 +48,7 @@ pub fn initialize_tracing( hostname: &str, mode: &Mode, environment: &str, -) -> (WorkerGuard, crate::otel_ee::OtelProvider) { +) -> (WorkerGuard, crate::otel_oss::OtelProvider) { let style = std::env::var("RUST_LOG_STYLE").unwrap_or_else(|_| "auto".into()); let rust_log_env = std::env::var("RUST_LOG"); @@ -70,16 +70,16 @@ pub fn initialize_tracing( LevelFilter::INFO }; - let meter_provider = crate::otel_ee::init_meter_provider(mode, hostname, environment); + let meter_provider = crate::otel_oss::init_meter_provider(mode, hostname, environment); #[cfg(all(feature = "otel", feature = "enterprise"))] - let opentelemetry = crate::otel_ee::init_otlp_tracer(mode, hostname, environment) + let opentelemetry = crate::otel_oss::init_otlp_tracer(mode, hostname, environment) .map(|x| tracing_opentelemetry::layer().with_tracer(x)); #[cfg(not(all(feature = "otel", feature = "enterprise")))] let opentelemetry: Option = None; - let logs_bridge = crate::otel_ee::init_logs_bridge(&mode, hostname, environment); + let logs_bridge = crate::otel_oss::init_logs_bridge(&mode, hostname, environment); use tracing_appender::rolling::{RollingFileAppender, Rotation}; diff --git a/backend/windmill-common/src/utils.rs b/backend/windmill-common/src/utils.rs index 3f1a1bd385..6b539dc73f 100644 --- a/backend/windmill-common/src/utils.rs +++ b/backend/windmill-common/src/utils.rs @@ -7,9 +7,9 @@ */ use crate::auth::is_devops_email; -use crate::ee::LICENSE_KEY_ID; +use crate::ee_oss::LICENSE_KEY_ID; #[cfg(feature = "enterprise")] -use crate::ee::{send_critical_alert, CriticalAlertKind}; +use crate::ee_oss::{send_critical_alert, CriticalAlertKind}; use crate::error::{to_anyhow, Error, Result}; use crate::global_settings::UNIQUE_ID_SETTING; use crate::DB; @@ -22,10 +22,14 @@ use croner::Cron; use rand::{distr::Alphanumeric, rng, Rng}; use reqwest::Client; use semver::Version; -use serde::{Deserialize, Deserializer, Serialize}; +use serde::{de::Error as SerdeDeserializerError, Deserialize, Deserializer, Serialize}; use sha2::{Digest, Sha256}; use sqlx::{Pool, Postgres}; -use std::str::FromStr; +use std::borrow::Cow; +use std::fmt::Display; +use std::{fs::DirBuilder as SyncDirBuilder, str::FromStr}; +use tokio::fs::DirBuilder as AsyncDirBuilder; +use url::Url; pub const MAX_PER_PAGE: usize = 10000; pub const DEFAULT_PER_PAGE: usize = 1000; @@ -33,6 +37,10 @@ pub const DEFAULT_PER_PAGE: usize = 1000; pub const GIT_VERSION: &str = git_version!(args = ["--tag", "--always"], fallback = "unknown-version"); +pub const AGENT_JWT_PREFIX: &str = "jwt_agent_"; +pub const WORKER_NAME_PREFIX: &str = "wk"; +pub const AGENT_WORKER_NAME_PREFIX: &str = "ag"; + use crate::CRITICAL_ALERT_MUTE_UI_ENABLED; use std::panic::{self, AssertUnwindSafe, Location}; use std::sync::atomic::Ordering; @@ -53,6 +61,12 @@ lazy_static::lazy_static! { } ).unwrap_or(Version::new(0, 1, 0)); + pub static ref HOSTNAME :String = std::env::var("FORCE_HOSTNAME").unwrap_or_else(|_| { + gethostname() + .to_str() + .map(|x| x.to_string()) + .unwrap_or_else(|| rd_string(5)) + }); pub static ref MODE_AND_ADDONS: ModeAndAddons = { let mut search_addon = false; @@ -70,7 +84,7 @@ lazy_static::lazy_static! { } Mode::Worker } else if &x == "agent" { - println!("Binary is in 'agent' mode"); + println!("Binary is in 'agent' mode with BASE_INTERNAL_URL={}", std::env::var("BASE_INTERNAL_URL").unwrap_or_default()); if std::env::var("BASE_INTERNAL_URL").is_err() { panic!("BASE_INTERNAL_URL is required in agent mode") } @@ -120,6 +134,10 @@ lazy_static::lazy_static! { }; } +lazy_static::lazy_static! { + pub static ref AGENT_TOKEN: String = std::env::var("AGENT_TOKEN").unwrap_or_default(); +} + #[derive(Clone)] pub struct ModeAndAddons { pub indexer: bool, @@ -167,15 +185,6 @@ pub async fn require_admin_or_devops( Ok(()) } -pub fn hostname() -> String { - std::env::var("FORCE_HOSTNAME").unwrap_or_else(|_| { - gethostname() - .to_str() - .map(|x| x.to_string()) - .unwrap_or_else(|| rd_string(5)) - }) -} - fn instance_name(hostname: &str) -> String { hostname .replace(" ", "") @@ -186,15 +195,30 @@ fn instance_name(hostname: &str) -> String { .to_string() } -pub fn worker_suffix(hostname: &str, rd_string: &str) -> String { - format!("{}-{}", instance_name(hostname), rd_string) +const DEFAULT_WORKER_SUFFIX_LEN: usize = 5; +pub const SSH_AGENT_WORKER_SUFFIX: &'static str = "/ssh"; + +pub fn create_worker_suffix(hostname: &str, rd_string_len: usize, ssh_ag_worker: bool) -> String { + let mut wk_suffix = format!("{}-{}", instance_name(hostname), rd_string(rd_string_len)); + if ssh_ag_worker { + wk_suffix.push_str(SSH_AGENT_WORKER_SUFFIX); + } + wk_suffix +} + +pub fn create_ssh_agent_worker_suffix(hostname: &str) -> String { + create_worker_suffix(hostname, DEFAULT_WORKER_SUFFIX_LEN, true) +} + +pub fn create_default_worker_suffix(hostname: &str) -> String { + create_worker_suffix(hostname, DEFAULT_WORKER_SUFFIX_LEN, false) } pub fn worker_name_with_suffix(is_agent: bool, worker_group: &str, suffix: &str) -> String { if is_agent { - format!("ag-{}-{}", worker_group, suffix) + format!("{}-{}-{}", AGENT_WORKER_NAME_PREFIX, worker_group, suffix) } else { - format!("wk-{}-{}", worker_group, suffix) + format!("{}-{}-{}", WORKER_NAME_PREFIX, worker_group, suffix) } } @@ -223,6 +247,21 @@ pub async fn now_from_db<'c, E: sqlx::PgExecutor<'c>>( .unwrap()) } +pub async fn create_directory_async(directory_path: &str) { + AsyncDirBuilder::new() + .recursive(true) + .create(directory_path) + .await + .expect("could not create dir"); +} + +pub fn create_directory_sync(directory_path: &str) { + SyncDirBuilder::new() + .recursive(true) + .create(directory_path) + .expect("could not create dir"); +} + pub fn not_found_if_none>(opt: Option, kind: &str, name: U) -> Result { if let Some(o) = opt { Ok(o) @@ -487,6 +526,18 @@ impl IsEmpty for Vec { } } +impl IsEmpty for Option +where + T: IsEmpty, +{ + fn is_empty(&self) -> bool { + match self { + Some(v) => v.is_empty(), + None => true, + } + } +} + pub fn empty_as_none<'de, D, T>(deserializer: D) -> std::result::Result, D::Error> where D: Deserializer<'de>, @@ -496,6 +547,26 @@ where Ok(option.filter(|s| !s.is_empty())) } +pub fn is_empty(value: &T) -> bool +where + T: IsEmpty, +{ + value.is_empty() +} + +pub fn deserialize_url<'de, D: Deserializer<'de>>( + de: D, +) -> std::result::Result, D::Error> { + let intermediate = >>::deserialize(de)?; + + match intermediate.as_deref() { + None | Some("") => Ok(None), + Some(non_empty_string) => Url::parse(non_empty_string) + .map(Some) + .map_err(D::Error::custom), + } +} + pub async fn fetch_mute_workspace(_db: &DB, workspace_id: &str) -> Result { match sqlx::query!( "SELECT mute_critical_alerts FROM workspace_settings WHERE workspace_id = $1", @@ -747,3 +818,20 @@ impl Future for WarnAfterFuture { } } } + +#[derive(Debug, Deserialize, Clone, Copy, PartialEq, Eq, Hash)] +#[serde(rename_all = "lowercase")] +pub enum RunnableKind { + Script, + Flow, +} + +impl Display for RunnableKind { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let runnable_kind = match self { + RunnableKind::Script => "script", + RunnableKind::Flow => "flow", + }; + write!(f, "{}", runnable_kind) + } +} diff --git a/backend/windmill-common/src/worker.rs b/backend/windmill-common/src/worker.rs index 6b11fa086f..1f09176512 100644 --- a/backend/windmill-common/src/worker.rs +++ b/backend/windmill-common/src/worker.rs @@ -1,4 +1,5 @@ use anyhow::anyhow; +use axum::http::HeaderMap; use bytes::Bytes; use const_format::concatcp; use itertools::Itertools; @@ -34,7 +35,6 @@ use crate::{ pub const DEFAULT_CLOUD_TIMEOUT: u64 = 900; pub const DEFAULT_SELFHOSTED_TIMEOUT: u64 = 604800; // 7 days - lazy_static::lazy_static! { pub static ref WORKER_GROUP: String = std::env::var("WORKER_GROUP").unwrap_or_else(|_| { #[cfg(not(feature = "enterprise"))] @@ -129,7 +129,7 @@ lazy_static::lazy_static! { pub static ref ALL_TAGS: Arc>> = Arc::new(RwLock::new(vec![])); - static ref CUSTOM_TAG_REGEX: Regex = Regex::new(r"^([\w-]+)\(((?:[\w-])+\+?)+\)$").unwrap(); + static ref CUSTOM_TAG_REGEX: Regex = Regex::new(r"^([\w-]+)\(((?:[\w-]+\+)*[\w-]+)\)$").unwrap(); pub static ref DISABLE_BUNDLING: bool = std::env::var("DISABLE_BUNDLING") .ok() @@ -159,12 +159,20 @@ impl HttpClient { pub async fn post( &self, url: &str, + headers: Option, body: &T, ) -> anyhow::Result { - let response = self + let response_builder = self .0 .post(format!("{}{}", *BASE_INTERNAL_URL, url)) - .json(body) + .json(body); + + let response_builder = match headers { + Some(headers) => response_builder.headers(headers), + None => response_builder, + }; + + let response = response_builder .send() .await .map_err(|e| anyhow::anyhow!(e))?; @@ -259,7 +267,7 @@ fn format_pull_query(peek: String) -> String { id, workspace_id, parent_job, created_by, created_at, runnable_id, runnable_path, args, kind, trigger, trigger_kind, permissioned_as, permissioned_as_email, script_lang, - flow_innermost_root_job, flow_step_id, + flow_innermost_root_job, flow_step_id, same_worker, pre_run_error, visible_to_owner, tag, concurrent_limit, concurrency_time_window_s, timeout, cache_ttl, priority, raw_code, raw_lock, raw_flow, script_entrypoint_override, preprocessed @@ -267,13 +275,13 @@ fn format_pull_query(peek: String) -> String { WHERE id = (SELECT id FROM peek) ) SELECT j.id, j.workspace_id, j.parent_job, j.created_by, started_at, scheduled_for, j.runnable_id, j.runnable_path, j.args, canceled_by, - canceled_reason, j.kind, j.trigger, j.trigger_kind, j.permissioned_as, + canceled_reason, j.kind, j.trigger, j.trigger_kind, j.permissioned_as, flow_status, j.script_lang, - j.same_worker, j.pre_run_error, j.visible_to_owner, + j.same_worker, j.pre_run_error, j.visible_to_owner, j.tag, j.concurrent_limit, j.concurrency_time_window_s, j.flow_innermost_root_job, j.timeout, j.flow_step_id, j.cache_ttl, j.priority, j.raw_code, j.raw_lock, j.raw_flow, j.script_entrypoint_override, j.preprocessed, pj.runnable_path as parent_runnable_path, - COALESCE(p.email, j.permissioned_as_email) as permissioned_as_email, p.username as permissioned_as_username, p.is_admin as permissioned_as_is_admin, + COALESCE(p.email, j.permissioned_as_email) as permissioned_as_email, p.username as permissioned_as_username, p.is_admin as permissioned_as_is_admin, p.is_operator as permissioned_as_is_operator, p.groups as permissioned_as_groups, p.folders as permissioned_as_folders FROM q, j LEFT JOIN v2_job_status f USING (id) @@ -308,7 +316,7 @@ pub async fn store_suspended_pull_query(wc: &WorkerConfig) { } pub fn make_pull_query(tags: &[String]) -> String { - format_pull_query(format!( + let query = format_pull_query(format!( "SELECT id FROM v2_job_queue WHERE running = false AND tag IN ({}) AND scheduled_for <= now() @@ -316,7 +324,8 @@ pub fn make_pull_query(tags: &[String]) -> String { FOR UPDATE SKIP LOCKED LIMIT 1", tags.iter().map(|x| format!("'{x}'")).join(", ") - )) + )); + query } pub async fn store_pull_query(wc: &WorkerConfig) { @@ -1028,7 +1037,7 @@ pub async fn update_ping_http( Ok(()) } -#[derive(Serialize, Deserialize)] +#[derive(Debug, Serialize, Deserialize)] pub struct JobCancelled { pub canceled_by: String, pub reason: String, @@ -1060,11 +1069,11 @@ pub async fn update_ping_for_failed_init_script_query( db: &DB, ) -> anyhow::Result<()> { sqlx::query!( - "UPDATE worker_ping SET -ping_at = now(), -jobs_executed = 1, -current_job_id = $1, -current_job_workspace_id = 'admins' + "UPDATE worker_ping SET +ping_at = now(), +jobs_executed = 1, +current_job_id = $1, +current_job_workspace_id = 'admins' WHERE worker = $2", last_job_id, worker_name @@ -1510,7 +1519,7 @@ pub struct WorkerConfig { impl std::fmt::Debug for WorkerConfig { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "WorkerConfig {{ worker_tags: {:?}, priority_tags_sorted: {:?}, dedicated_worker: {:?}, init_bash: {:?}, cache_clear: {:?}, additional_python_paths: {:?}, pip_local_dependencies: {:?}, env_vars: {:?} }}", + write!(f, "WorkerConfig {{ worker_tags: {:?}, priority_tags_sorted: {:?}, dedicated_worker: {:?}, init_bash: {:?}, cache_clear: {:?}, additional_python_paths: {:?}, pip_local_dependencies: {:?}, env_vars: {:?} }}", self.worker_tags, self.priority_tags_sorted, self.dedicated_worker, self.init_bash, self.cache_clear, self.additional_python_paths, self.pip_local_dependencies, self.env_vars.iter().map(|(k, v)| format!("{}: {}{} ({} chars)", k, &v[..3.min(v.len())], "***", v.len())).collect::>().join(", ")) } } diff --git a/backend/windmill-git-sync/Cargo.toml b/backend/windmill-git-sync/Cargo.toml index d5a8412fae..6a0499639a 100644 --- a/backend/windmill-git-sync/Cargo.toml +++ b/backend/windmill-git-sync/Cargo.toml @@ -9,6 +9,7 @@ name = "windmill_git_sync" path = "./src/lib.rs" [features] +private = [] enterprise = ["windmill-queue/enterprise", "windmill-common/enterprise"] default = [] diff --git a/backend/windmill-git-sync/src/git_sync_ee.rs b/backend/windmill-git-sync/src/git_sync_oss.rs similarity index 69% rename from backend/windmill-git-sync/src/git_sync_ee.rs rename to backend/windmill-git-sync/src/git_sync_oss.rs index cc245d3d0c..70e2b53640 100644 --- a/backend/windmill-git-sync/src/git_sync_ee.rs +++ b/backend/windmill-git-sync/src/git_sync_oss.rs @@ -1,7 +1,14 @@ +#[cfg(feature = "private")] +#[allow(unused)] +pub use crate::git_sync_ee::*; + +#[cfg(not(feature = "private"))] use windmill_common::error::Result; +#[cfg(not(feature = "private"))] use crate::{DeployedObject, DB}; +#[cfg(not(feature = "private"))] pub async fn handle_deployment_metadata<'c>( _email: &str, _created_by: &str, diff --git a/backend/windmill-git-sync/src/lib.rs b/backend/windmill-git-sync/src/lib.rs index f1e0ed6ee3..d4e4ebf71f 100644 --- a/backend/windmill-git-sync/src/lib.rs +++ b/backend/windmill-git-sync/src/lib.rs @@ -10,9 +10,11 @@ use sqlx::{Pool, Postgres}; use windmill_common::scripts::ScriptHash; +#[cfg(feature = "private")] pub mod git_sync_ee; +pub mod git_sync_oss; -pub use git_sync_ee::handle_deployment_metadata; +pub use git_sync_oss::handle_deployment_metadata; pub type DB = Pool; #[derive(Clone, Debug)] diff --git a/backend/windmill-indexer/Cargo.toml b/backend/windmill-indexer/Cargo.toml index 14ab53beef..ea95d6237d 100644 --- a/backend/windmill-indexer/Cargo.toml +++ b/backend/windmill-indexer/Cargo.toml @@ -11,6 +11,7 @@ path = "src/lib.rs" [features] default = [] parquet = ["dep:object_store"] +private = [] enterprise = [] [dependencies] diff --git a/backend/windmill-indexer/src/completed_runs_ee.rs b/backend/windmill-indexer/src/completed_runs_oss.rs similarity index 66% rename from backend/windmill-indexer/src/completed_runs_ee.rs rename to backend/windmill-indexer/src/completed_runs_oss.rs index f5c6c98cf8..083f8744fa 100644 --- a/backend/windmill-indexer/src/completed_runs_ee.rs +++ b/backend/windmill-indexer/src/completed_runs_oss.rs @@ -1,17 +1,28 @@ +#[cfg(feature = "private")] +#[allow(unused)] +pub use crate::completed_runs_ee::*; + +#[cfg(not(feature = "private"))] use anyhow::anyhow; +#[cfg(not(feature = "private"))] use sqlx::{Pool, Postgres}; +#[cfg(not(feature = "private"))] use windmill_common::error::Error; +#[cfg(not(feature = "private"))] #[derive(Clone)] pub struct IndexReader; +#[cfg(not(feature = "private"))] #[derive(Clone)] pub struct IndexWriter; +#[cfg(not(feature = "private"))] pub async fn init_index(_db: &Pool) -> Result<(IndexReader, IndexWriter), Error> { Err(anyhow!("Cannot initialize index: not in EE").into()) } +#[cfg(not(feature = "private"))] pub async fn run_indexer( _db: Pool, mut _index_writer: IndexWriter, diff --git a/backend/windmill-indexer/src/indexer_ee.rs b/backend/windmill-indexer/src/indexer_ee.rs deleted file mode 100644 index 8b13789179..0000000000 --- a/backend/windmill-indexer/src/indexer_ee.rs +++ /dev/null @@ -1 +0,0 @@ - diff --git a/backend/windmill-indexer/src/indexer_oss.rs b/backend/windmill-indexer/src/indexer_oss.rs new file mode 100644 index 0000000000..198c14e526 --- /dev/null +++ b/backend/windmill-indexer/src/indexer_oss.rs @@ -0,0 +1,5 @@ +#[cfg(feature = "private")] +#[allow(unused)] +pub use crate::indexer_ee::*; + + diff --git a/backend/windmill-indexer/src/lib.rs b/backend/windmill-indexer/src/lib.rs index 59c6a627f7..6c13b551d1 100644 --- a/backend/windmill-indexer/src/lib.rs +++ b/backend/windmill-indexer/src/lib.rs @@ -1,3 +1,9 @@ +#[cfg(feature = "private")] pub mod completed_runs_ee; +pub mod completed_runs_oss; +#[cfg(feature = "private")] pub mod indexer_ee; +pub mod indexer_oss; +#[cfg(feature = "private")] pub mod service_logs_ee; +pub mod service_logs_oss; diff --git a/backend/windmill-indexer/src/service_logs_ee.rs b/backend/windmill-indexer/src/service_logs_oss.rs similarity index 68% rename from backend/windmill-indexer/src/service_logs_ee.rs rename to backend/windmill-indexer/src/service_logs_oss.rs index e04575d3f5..b6e9017f88 100644 --- a/backend/windmill-indexer/src/service_logs_ee.rs +++ b/backend/windmill-indexer/src/service_logs_oss.rs @@ -1,13 +1,24 @@ +#[cfg(feature = "private")] +#[allow(unused)] +pub use crate::service_logs_ee::*; + +#[cfg(not(feature = "private"))] use anyhow::anyhow; +#[cfg(not(feature = "private"))] use sqlx::{Pool, Postgres}; +#[cfg(not(feature = "private"))] use windmill_common::error::Error; +#[cfg(not(feature = "private"))] use windmill_common::KillpillSender; #[derive(Clone)] +#[cfg(not(feature = "private"))] pub struct ServiceLogIndexReader; #[derive(Clone)] +#[cfg(not(feature = "private"))] pub struct ServiceLogIndexWriter; +#[cfg(not(feature = "private"))] pub async fn init_index( _db: &Pool, mut _killpill_tx: KillpillSender, @@ -15,6 +26,7 @@ pub async fn init_index( Err(anyhow!("Cannot initialize index: not in EE").into()) } +#[cfg(not(feature = "private"))] pub async fn run_indexer( _db: Pool, mut _index_writer: ServiceLogIndexWriter, diff --git a/backend/windmill-queue/Cargo.toml b/backend/windmill-queue/Cargo.toml index a90fa6ccf7..4458e8aa99 100644 --- a/backend/windmill-queue/Cargo.toml +++ b/backend/windmill-queue/Cargo.toml @@ -10,6 +10,7 @@ path = "src/lib.rs" [features] default = [] +private = [] enterprise = ["windmill-common/enterprise"] cloud = [] benchmark = ["windmill-common/benchmark"] diff --git a/backend/windmill-queue/src/jobs.rs b/backend/windmill-queue/src/jobs.rs index de64071c42..1676148b00 100644 --- a/backend/windmill-queue/src/jobs.rs +++ b/backend/windmill-queue/src/jobs.rs @@ -25,7 +25,7 @@ use sqlx::{types::Json, FromRow, Pool, Postgres, Transaction}; use tokio::{sync::RwLock, time::sleep}; use ulid::Ulid; use uuid::Uuid; -use windmill_audit::audit_ee::{audit_log, AuditAuthor}; +use windmill_audit::audit_oss::{audit_log, AuditAuthor}; use windmill_audit::ActionKind; #[cfg(feature = "benchmark")] @@ -71,7 +71,7 @@ use windmill_common::BASE_URL; use windmill_common::users::SUPERADMIN_SYNC_EMAIL; use crate::flow_status::{update_flow_status_in_progress, update_workflow_as_code_status}; -use crate::jobs_ee::update_concurrency_counter; +use crate::jobs_oss::update_concurrency_counter; use crate::schedule::{get_schedule_opt, push_scheduled_job}; use crate::tags::per_workspace_tag; @@ -391,6 +391,7 @@ pub async fn append_logs( if let Err(e) = client .post::<_, String>( &format!("/api/w/{}/agent_workers/push_logs/{}", workspace.as_ref(), job_id), + None, &logs.as_ref(), ) .await { @@ -1010,33 +1011,43 @@ pub async fn add_completed_job( #[cfg(feature = "cloud")] if *CLOUD_HOSTED && !queued_job.is_flow() && _duration > 1000 { - let additional_usage = _duration / 1000; - let w_id = &queued_job.workspace_id; - let premium_workspace = windmill_common::workspaces::is_premium_workspace(db, w_id).await; - let _ = sqlx::query!( - "INSERT INTO usage (id, is_workspace, month_, usage) - VALUES ($1, TRUE, EXTRACT(YEAR FROM current_date) * 12 + EXTRACT(MONTH FROM current_date), $2) - ON CONFLICT (id, is_workspace, month_) DO UPDATE SET usage = usage.usage + $2", - w_id, - additional_usage as i32 - ) - .execute(db) - .await - .map_err(|e| Error::internal_err(format!("updating usage: {e:#}"))); - - if !premium_workspace { + let db = db.clone(); + let w_id = queued_job.workspace_id.clone(); + let email = queued_job.permissioned_as_email.clone(); + let w_id2 = w_id.clone(); + let email2 = email.clone(); + tokio::task::spawn(async move { + let additional_usage = _duration / 1000; + let premium_workspace = windmill_common::workspaces::is_premium_workspace(&db, &w_id).await; + tokio::time::timeout(std::time::Duration::from_secs(10), async move { let _ = sqlx::query!( "INSERT INTO usage (id, is_workspace, month_, usage) - VALUES ($1, FALSE, EXTRACT(YEAR FROM current_date) * 12 + EXTRACT(MONTH FROM current_date), $2) + VALUES ($1, TRUE, EXTRACT(YEAR FROM current_date) * 12 + EXTRACT(MONTH FROM current_date), $2) ON CONFLICT (id, is_workspace, month_) DO UPDATE SET usage = usage.usage + $2", - queued_job.permissioned_as_email, + w_id, additional_usage as i32 ) - .execute(db) + .execute(&db) .await .map_err(|e| Error::internal_err(format!("updating usage: {e:#}"))); - } + + if !premium_workspace { + let _ = sqlx::query!( + "INSERT INTO usage (id, is_workspace, month_, usage) + VALUES ($1, FALSE, EXTRACT(YEAR FROM current_date) * 12 + EXTRACT(MONTH FROM current_date), $2) + ON CONFLICT (id, is_workspace, month_) DO UPDATE SET usage = usage.usage + $2", + email, + additional_usage as i32 + ) + .execute(&db) + .await + .map_err(|e| Error::internal_err(format!("updating usage: {e:#}"))); + }}).await.unwrap_or_else(|_| { + tracing::error!("Could not update usage for workspace {w_id2} and permissioned as {email2}, stopped after 10s"); + }); + }); } + #[cfg(feature = "enterprise")] if !success { @@ -2127,7 +2138,7 @@ pub struct PulledJob { // NOTE: // Precomputed by the server // Used to offload work from agent workers to server -#[derive(Serialize, Deserialize)] +#[derive(Debug, Serialize, Deserialize)] pub enum PrecomputedAgentInfo { Bun { local: String, remote: String }, Python { @@ -2138,7 +2149,7 @@ pub enum PrecomputedAgentInfo { requirements: Option }, } -#[derive(Serialize, Deserialize)] +#[derive(Debug, Serialize, Deserialize)] pub struct JobAndPerms { pub job: MiniPulledJob, pub raw_code: Option, @@ -2290,16 +2301,20 @@ pub async fn pull( db: &Pool, suspend_first: bool, worker_name: &str, - query_o: Option<(String, String)>, + query_o: Option<&(String, String)>, #[cfg(feature = "benchmark")] bench: &mut BenchmarkIter, ) -> windmill_common::error::Result { loop { - if let Some((query_suspended, query_no_suspend)) = query_o.as_ref() { + if let Some((query_suspended, query_no_suspend)) = query_o { let njob = { - let job = sqlx::query_as::<_, PulledJob>(query_suspended) + let job = if query_suspended.is_empty() { + None + } else { + sqlx::query_as::<_, PulledJob>(query_suspended) .bind(worker_name) .fetch_optional(db) - .await?; + .await? + }; if let Some(job) = job { PulledJobResult { job: Some(job), suspended: true } } else { @@ -2549,7 +2564,6 @@ async fn pull_single_job_and_mark_as_running_no_concurrency_limit<'c>( tokio::time::sleep(tokio::time::Duration::from_secs(1)).await; return Ok((None, false)); } - let r = if suspend_first { // tracing::info!("Pulling job with query: {}", query); sqlx::query_as::<_, PulledJob>(&query) @@ -2702,7 +2716,7 @@ pub async fn get_result_by_id( .await { Ok(res) => Ok(res), - Err(_) => { + Err(e) => { let root = sqlx::query!( "SELECT id As \"id!\", @@ -2719,7 +2733,7 @@ pub async fn get_result_by_id( let restarted_from_id = not_found_if_none( root.restarted_from, "Id not found in the result's mapping of the root job and root job had no restarted from information", - format!("parent: {}, root: {}, id: {}", flow_id, root.id, node_id), + format!("parent: {}, root: {}, id: {}, error: {e:#}", flow_id, root.id, node_id), )?; get_result_by_id_from_original_flow( @@ -3265,35 +3279,40 @@ pub async fn push<'c, 'd>( job_payload, JobPayload::Flow { .. } | JobPayload::RawFlow { .. } ) { - let workspace_usage = sqlx::query_scalar!( - "INSERT INTO usage (id, is_workspace, month_, usage) - VALUES ($1, TRUE, EXTRACT(YEAR FROM current_date) * 12 + EXTRACT(MONTH FROM current_date), 1) - ON CONFLICT (id, is_workspace, month_) DO UPDATE SET usage = usage.usage + 1 - RETURNING usage.usage", - workspace_id - ) - .fetch_one(_db) - .await - .map_err(|e| Error::internal_err(format!("updating usage: {e:#}")))?; + tokio::time::timeout(std::time::Duration::from_secs(10), async move { + let workspace_usage = sqlx::query_scalar!( + "INSERT INTO usage (id, is_workspace, month_, usage) + VALUES ($1, TRUE, EXTRACT(YEAR FROM current_date) * 12 + EXTRACT(MONTH FROM current_date), 1) + ON CONFLICT (id, is_workspace, month_) DO UPDATE SET usage = usage.usage + 1 + RETURNING usage.usage", + workspace_id + ) + .fetch_one(_db) + .await + .map_err(|e| Error::internal_err(format!("updating usage: {e:#}")))?; - let user_usage = if !premium_workspace { - Some(sqlx::query_scalar!( - "INSERT INTO usage (id, is_workspace, month_, usage) - VALUES ($1, FALSE, EXTRACT(YEAR FROM current_date) * 12 + EXTRACT(MONTH FROM current_date), 1) - ON CONFLICT (id, is_workspace, month_) DO UPDATE SET usage = usage.usage + 1 - RETURNING usage.usage", - email - ) - .fetch_one(_db) - .await - .map_err(|e| Error::internal_err(format!("updating usage: {e:#}")))?) - } else { - None - }; - (Some(workspace_usage), user_usage) + let user_usage = if !premium_workspace { + Some(sqlx::query_scalar!( + "INSERT INTO usage (id, is_workspace, month_, usage) + VALUES ($1, FALSE, EXTRACT(YEAR FROM current_date) * 12 + EXTRACT(MONTH FROM current_date), 1) + ON CONFLICT (id, is_workspace, month_) DO UPDATE SET usage = usage.usage + 1 + RETURNING usage.usage", + email + ) + .fetch_one(_db) + .await + .map_err(|e| Error::internal_err(format!("updating usage: {e:#}")))?) + } else { + None + }; + Ok((Some(workspace_usage), user_usage)) + }).await.unwrap_or_else(|e| { + tracing::error!("Could not update usage for workspace {workspace_id} and permissioned as {email}, stopped after 10s: {e:#}"); + Err(Error::internal_err(format!("Could not update usage for workspace {workspace_id} and permissioned as {email}, stopped after 10s: {e:#}"))) + }) } else { - (None, None) - }; + Ok((None, None)) + }?; if !premium_workspace { let is_super_admin = diff --git a/backend/windmill-queue/src/jobs_ee.rs b/backend/windmill-queue/src/jobs_oss.rs similarity index 67% rename from backend/windmill-queue/src/jobs_ee.rs rename to backend/windmill-queue/src/jobs_oss.rs index ea1f86b529..987a56bf0d 100644 --- a/backend/windmill-queue/src/jobs_ee.rs +++ b/backend/windmill-queue/src/jobs_oss.rs @@ -1,7 +1,15 @@ +#[cfg(feature = "private")] +#[allow(unused)] +pub use crate::jobs_ee::*; + +#[cfg(not(feature = "private"))] use chrono::{DateTime, Utc}; +#[cfg(not(feature = "private"))] use uuid::Uuid; +#[cfg(not(feature = "private"))] use windmill_common::DB; +#[cfg(not(feature = "private"))] #[allow(dead_code)] pub(crate) async fn update_concurrency_counter( _db: &DB, diff --git a/backend/windmill-queue/src/lib.rs b/backend/windmill-queue/src/lib.rs index 2496bdf818..9b6025e515 100644 --- a/backend/windmill-queue/src/lib.rs +++ b/backend/windmill-queue/src/lib.rs @@ -7,7 +7,9 @@ */ mod jobs; +#[cfg(feature = "private")] pub mod jobs_ee; +pub mod jobs_oss; pub mod schedule; pub use jobs::*; pub mod flow_status; diff --git a/backend/windmill-queue/src/schedule.rs b/backend/windmill-queue/src/schedule.rs index bb7d5df906..294ece7509 100644 --- a/backend/windmill-queue/src/schedule.rs +++ b/backend/windmill-queue/src/schedule.rs @@ -13,7 +13,7 @@ use sqlx::{PgExecutor, Postgres, Transaction}; use std::collections::HashMap; use std::str::FromStr; use windmill_common::db::Authed; -use windmill_common::ee::LICENSE_KEY_VALID; +use windmill_common::ee_oss::LICENSE_KEY_VALID; use windmill_common::flows::Retry; use windmill_common::get_latest_flow_version_info_for_path; use windmill_common::jobs::JobPayload; diff --git a/backend/windmill-worker/Cargo.toml b/backend/windmill-worker/Cargo.toml index 6a6a0abf01..a3b36a5c04 100644 --- a/backend/windmill-worker/Cargo.toml +++ b/backend/windmill-worker/Cargo.toml @@ -10,6 +10,7 @@ path = "src/lib.rs" [features] default = [] +private = [] prometheus = ["dep:prometheus", "windmill-common/prometheus"] enterprise = ["windmill-queue/enterprise", "windmill-git-sync/enterprise", "windmill-common/enterprise", "dep:pem", "dep:tokio-util"] mssql = ["dep:tiberius"] diff --git a/backend/windmill-worker/src/agent_workers.rs b/backend/windmill-worker/src/agent_workers.rs index 192ae7ae37..f5f5a3b0cc 100644 --- a/backend/windmill-worker/src/agent_workers.rs +++ b/backend/windmill-worker/src/agent_workers.rs @@ -1,3 +1,4 @@ +use reqwest::header::HeaderMap; use uuid::Uuid; use windmill_common::{agent_workers::QueueInitJob, worker::HttpClient}; use windmill_queue::{JobAndPerms, JobCompleted}; @@ -6,14 +7,21 @@ pub async fn queue_init_job(client: &HttpClient, content: &str) -> anyhow::Resul client .post( "/api/agent_workers/queue_init_job", + None, &QueueInitJob { content: content.to_string() }, ) .await .and_then(|x: String| Uuid::parse_str(&x).map_err(|e| anyhow::anyhow!(e))) } -pub async fn pull_job(client: &HttpClient) -> anyhow::Result> { - client.post("/api/agent_workers/pull_job", &()).await +pub async fn pull_job( + client: &HttpClient, + headers: Option, + body: Option, +) -> anyhow::Result> { + client + .post("/api/agent_workers/pull_job", headers, &body) + .await } pub async fn send_result(client: &HttpClient, jc: JobCompleted) -> anyhow::Result { @@ -23,6 +31,7 @@ pub async fn send_result(client: &HttpClient, jc: JobCompleted) -> anyhow::Resul "/api/w/{}/agent_workers/send_result/{}", jc.job.workspace_id, jc.job.id ), + None, &jc, ) .await diff --git a/backend/windmill-worker/src/bash_executor.rs b/backend/windmill-worker/src/bash_executor.rs index ec08003dd3..20b8fd0d8f 100644 --- a/backend/windmill-worker/src/bash_executor.rs +++ b/backend/windmill-worker/src/bash_executor.rs @@ -43,12 +43,11 @@ use crate::{ OccupancyMetrics, }, handle_child::handle_child, - DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV, NSJAIL_PATH, PATH_ENV, - POWERSHELL_CACHE_DIR, POWERSHELL_PATH, PROXY_ENVS, TZ_ENV, + DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV, NSJAIL_PATH, PATH_ENV, POWERSHELL_CACHE_DIR, + POWERSHELL_PATH, PROXY_ENVS, TZ_ENV, }; use windmill_common::client::AuthedClient; - #[cfg(windows)] use crate::SYSTEM_ROOT; @@ -275,6 +274,19 @@ exit $exit_status ))) } +#[cfg(feature = "dind")] +async fn rm_container(client: &bollard::Docker, container_id: &str) { + if let Err(e) = client + .remove_container( + container_id, + Some(RemoveContainerOptions { force: true, ..Default::default() }), + ) + .await + { + tracing::error!("Error removing container: {:?}", e); + } +} + #[cfg(feature = "dind")] async fn handle_docker_job( job_id: Uuid, @@ -446,19 +458,12 @@ async fn handle_docker_job( } } } + rm_container(&client, &container_id).await; return Err(e); } - if let Err(e) = client - .remove_container( - &container_id, - Some(RemoveContainerOptions { force: true, ..Default::default() }), - ) - .await - { - tracing::error!("Error removing container: {:?}", e); - } + rm_container(&client, &container_id).await; let result = result.unwrap(); diff --git a/backend/windmill-worker/src/common.rs b/backend/windmill-worker/src/common.rs index 9955256d13..b419626845 100644 --- a/backend/windmill-worker/src/common.rs +++ b/backend/windmill-worker/src/common.rs @@ -492,6 +492,7 @@ pub async fn update_worker_ping_for_failed_init_script( if let Err(e) = client .post::<_, ()>( UPDATE_PING_URL, + None, &Ping { last_job_executed: Some(last_job_id), last_job_workspace_id: None, @@ -728,7 +729,7 @@ async fn get_workspace_s3_resource_path( storage: Option<&String>, ) -> windmill_common::error::Result> { use windmill_common::{ - job_s3_helpers_ee::get_s3_resource_internal, s3_helpers::StorageResourceType, + job_s3_helpers_oss::get_s3_resource_internal, s3_helpers::StorageResourceType, }; let raw_lfs_opt = if let Some(storage) = storage { @@ -786,7 +787,7 @@ async fn get_workspace_s3_resource_path( get_s3_resource_internal( rt, s3_resource_value_raw, - windmill_common::job_s3_helpers_ee::TokenGenerator::AsClient(client), + windmill_common::job_s3_helpers_oss::TokenGenerator::AsClient(client), db, ) .await @@ -1209,8 +1210,8 @@ pub async fn par_install_language_dependencies<'a>( } #[cfg(all(feature = "enterprise", feature = "parquet"))] let is_not_pro = !matches!( - windmill_common::ee::get_license_plan().await, - windmill_common::ee::LicensePlan::Pro + windmill_common::ee_oss::get_license_plan().await, + windmill_common::ee_oss::LicensePlan::Pro ); #[cfg(all(feature = "enterprise", feature = "parquet"))] if is_not_pro && matches!(install_fn, InstallStrategy::AllAtOnce(_)) { diff --git a/backend/windmill-worker/src/go_executor.rs b/backend/windmill-worker/src/go_executor.rs index 2217c775ba..c5fe55184a 100644 --- a/backend/windmill-worker/src/go_executor.rs +++ b/backend/windmill-worker/src/go_executor.rs @@ -3,7 +3,11 @@ use std::{collections::HashMap, fs::DirBuilder, process::Stdio}; use itertools::Itertools; use serde_json::value::RawValue; -use tokio::{fs::File, io::AsyncReadExt, process::Command}; +use tokio::{ + fs::{self, File}, + io::AsyncReadExt, + process::Command, +}; use uuid::Uuid; use windmill_common::{ error::{self, Error}, @@ -91,6 +95,7 @@ pub async fn handle_go_job( true, skip_go_mod, skip_tidy, + false, worker_name, &job.workspace_id, occupation_metrics, @@ -356,11 +361,21 @@ pub async fn install_go_dependencies( non_dep_job: bool, skip_go_mod: bool, has_sum: bool, + raw_deps: bool, worker_name: &str, w_id: &str, occupation_metrics: &mut OccupancyMetrics, ) -> error::Result { - if !skip_go_mod { + if raw_deps { + let go_mod = + if let Some(module) = code.lines().find(|l| l.trim_start().starts_with("module ")) { + code.replace(module, "module mymod") + } else { + format!("module mymod\n{code}") + }; + fs::write(format!("{job_dir}/go.mod"), go_mod).await?; + } + if !raw_deps && !skip_go_mod { gen_go_mymod(code, job_dir).await?; let mut child_cmd = Command::new(GO_PATH.as_str()); child_cmd @@ -400,7 +415,9 @@ pub async fn install_go_dependencies( let mut new_lockfile = false; - let hash = if !has_sum { + let hash = if raw_deps { + calculate_hash(code) + } else if !has_sum { calculate_hash(parse_go_imports(&code)?.iter().join("\n").as_str()) } else { "".to_string() @@ -429,7 +446,15 @@ pub async fn install_go_dependencies( } } - let mod_command = if skip_tidy { "download" } else { "tidy" }; + let mod_command = if skip_tidy || + // If there is go.mod provided we want to use `download` only. + // Unlike `tidy` it does not modify local go.mod + raw_deps + { + "download" + } else { + "tidy" + }; let mut child_cmd = Command::new(GO_PATH.as_str()); child_cmd .current_dir(job_dir) diff --git a/backend/windmill-worker/src/handle_child.rs b/backend/windmill-worker/src/handle_child.rs index 3eb9e95dc9..467fac7c60 100644 --- a/backend/windmill-worker/src/handle_child.rs +++ b/backend/windmill-worker/src/handle_child.rs @@ -53,7 +53,7 @@ use futures::{ use crate::common::{resolve_job_timeout, OccupancyMetrics}; use crate::job_logger::{append_job_logs, append_with_limit}; -use crate::job_logger_ee::process_streaming_log_lines; +use crate::job_logger_oss::process_streaming_log_lines; use crate::worker_utils::{ping_job_status, update_worker_ping_from_job}; use crate::{MAX_RESULT_SIZE, MAX_WAIT_FOR_SIGINT, MAX_WAIT_FOR_SIGTERM}; @@ -227,6 +227,7 @@ pub async fn handle_child( if let Err(err) = client .post::<_, ()>( &format!("/api/agent_workers/set_job_cancelled/{}", job_id), + None, &JobCancelled { canceled_by: "timeout".to_string(), reason: format!("duration > {}", timeout_duration.as_secs()), diff --git a/backend/windmill-worker/src/job_logger.rs b/backend/windmill-worker/src/job_logger.rs index f4abd1d0c2..626dfbce44 100644 --- a/backend/windmill-worker/src/job_logger.rs +++ b/backend/windmill-worker/src/job_logger.rs @@ -13,10 +13,10 @@ use std::sync::Arc; use uuid::Uuid; #[cfg(not(all(feature = "enterprise", feature = "parquet")))] -use crate::job_logger_ee::default_disk_log_storage; +use crate::job_logger_oss::default_disk_log_storage; #[cfg(all(feature = "enterprise", feature = "parquet"))] -use crate::job_logger_ee::s3_storage; +use crate::job_logger_oss::s3_storage; pub enum CompactLogs { #[cfg(not(all(feature = "enterprise", feature = "parquet")))] diff --git a/backend/windmill-worker/src/job_logger_ee.rs b/backend/windmill-worker/src/job_logger_oss.rs similarity index 68% rename from backend/windmill-worker/src/job_logger_ee.rs rename to backend/windmill-worker/src/job_logger_oss.rs index 22772878ee..3ee3d29a3a 100644 --- a/backend/windmill-worker/src/job_logger_ee.rs +++ b/backend/windmill-worker/src/job_logger_oss.rs @@ -1,13 +1,14 @@ -use std::io; -use std::sync::atomic::AtomicU32; -use std::sync::Arc; +#[cfg(feature = "private")] +#[allow(unused)] +pub use crate::job_logger_ee::*; -use uuid::Uuid; -use windmill_common::DB; +#[cfg(not(feature = "private"))] +use { + crate::job_logger::CompactLogs, std::io, std::sync::atomic::AtomicU32, std::sync::Arc, + uuid::Uuid, windmill_common::DB, +}; -use crate::job_logger::CompactLogs; - -#[cfg(all(feature = "enterprise", feature = "parquet"))] +#[cfg(all(feature = "enterprise", feature = "parquet", not(feature = "private")))] pub(crate) async fn s3_storage( _job_id: &Uuid, _w_id: &str, @@ -19,6 +20,7 @@ pub(crate) async fn s3_storage( tracing::info!("Logs length of {_job_id} has exceeded a threshold. Implementation to store excess on s3 in not OSS"); } +#[cfg(not(feature = "private"))] #[allow(dead_code)] pub(crate) async fn default_disk_log_storage( job_id: &Uuid, @@ -32,6 +34,7 @@ pub(crate) async fn default_disk_log_storage( tracing::info!("Logs length of {job_id} has exceeded a threshold. Implementation to store excess on disk in not OSS"); } +#[cfg(not(feature = "private"))] pub(crate) fn process_streaming_log_lines( r: Result, io::Error>, _stderr: bool, diff --git a/backend/windmill-worker/src/lib.rs b/backend/windmill-worker/src/lib.rs index fa6eb10960..3dfeba5c42 100644 --- a/backend/windmill-worker/src/lib.rs +++ b/backend/windmill-worker/src/lib.rs @@ -27,7 +27,9 @@ mod go_executor; mod graphql_executor; mod handle_child; pub mod job_logger; -mod job_logger_ee; +#[cfg(feature = "private")] +pub mod job_logger_ee; +mod job_logger_oss; mod js_eval; #[cfg(feature = "mysql")] mod mysql_executor; @@ -35,7 +37,9 @@ mod mysql_executor; mod nu_executor; #[cfg(feature = "oracledb")] mod oracledb_executor; -mod otel_ee; +#[cfg(feature = "private")] +pub mod otel_ee; +mod otel_oss; mod pg_executor; #[cfg(feature = "php")] mod php_executor; @@ -53,9 +57,8 @@ mod worker_flow; mod worker_lockfiles; mod worker_utils; -pub use worker_lockfiles::process_relative_imports; - pub use worker::*; +pub use worker_lockfiles::process_relative_imports; pub use result_processor::handle_job_error; diff --git a/backend/windmill-worker/src/otel_ee.rs b/backend/windmill-worker/src/otel_ee.rs deleted file mode 100644 index 311ffd6355..0000000000 --- a/backend/windmill-worker/src/otel_ee.rs +++ /dev/null @@ -1,3 +0,0 @@ -use windmill_queue::MiniPulledJob; - -pub fn add_root_flow_job_to_otlp(_queued_job: &MiniPulledJob, _success: bool) {} diff --git a/backend/windmill-worker/src/otel_oss.rs b/backend/windmill-worker/src/otel_oss.rs new file mode 100644 index 0000000000..2f65c534b4 --- /dev/null +++ b/backend/windmill-worker/src/otel_oss.rs @@ -0,0 +1,9 @@ +#[cfg(feature = "private")] +#[allow(unused)] +pub use crate::otel_ee::*; + +#[cfg(not(feature = "private"))] +use windmill_queue::MiniPulledJob; + +#[cfg(not(feature = "private"))] +pub fn add_root_flow_job_to_otlp(_queued_job: &MiniPulledJob, _success: bool) {} diff --git a/backend/windmill-worker/src/python_executor.rs b/backend/windmill-worker/src/python_executor.rs index 76ee967656..35815e452c 100644 --- a/backend/windmill-worker/src/python_executor.rs +++ b/backend/windmill-worker/src/python_executor.rs @@ -20,7 +20,7 @@ use tokio::{ }; use uuid::Uuid; #[cfg(all(feature = "enterprise", feature = "parquet", unix))] -use windmill_common::ee::{get_license_plan, LicensePlan}; +use windmill_common::ee_oss::{get_license_plan, LicensePlan}; use windmill_common::{ error::{ self, @@ -658,7 +658,7 @@ except BaseException as e: // Add /tmp/windmill/cache/python_x_y_z/global-site-packages to PYTHONPATH. // Usefull if certain wheels needs to be preinstalled before execution. - let global_site_packages_path = py_version.to_cache_dir() + "/global-site-packages"; + let global_site_packages_path = py_version.to_cache_dir(true) + "/global-site-packages"; let additional_python_paths_folders = { let mut paths = additional_python_paths.clone(); if std::fs::metadata(&global_site_packages_path).is_ok() { @@ -1489,7 +1489,7 @@ pub async fn handle_python_reqs( if req.starts_with('#') || req.starts_with('-') || req.trim().is_empty() { continue; } - let py_prefix = &py_version.to_cache_dir(); + let py_prefix = &py_version.to_cache_dir(false); let venv_p = format!( "{py_prefix}/{}", @@ -1740,7 +1740,7 @@ pub async fn handle_python_reqs( tokio::select! { // Cancel was called on the job _ = kill_rx.recv() => return Err(anyhow::anyhow!("S3 pull was canceled")), - pull = pull_from_tar(os, venv_p.clone(), py_version.to_cache_dir_top_level(), None, false) => { + pull = pull_from_tar(os, venv_p.clone(), py_version.to_cache_dir_top_level(false), None, false) => { if let Err(e) = pull { tracing::info!( workspace_id = %w_id, @@ -1891,7 +1891,7 @@ pub async fn handle_python_reqs( #[cfg(all(feature = "enterprise", feature = "parquet", unix))] if s3_push { if let Some(os) = windmill_common::s3_helpers::get_object_store().await { - tokio::spawn(build_tar_and_push(os, venv_p.clone(), py_version.to_cache_dir_top_level(), None, false)); + tokio::spawn(build_tar_and_push(os, venv_p.clone(), py_version.to_cache_dir_top_level(false), None, false)); } } diff --git a/backend/windmill-worker/src/python_versions.rs b/backend/windmill-worker/src/python_versions.rs index 26be7aca65..2f12d34dc6 100644 --- a/backend/windmill-worker/src/python_versions.rs +++ b/backend/windmill-worker/src/python_versions.rs @@ -153,12 +153,18 @@ impl PyV { let valid = all_versions .clone() .into_iter() - .filter(|v| version_specifiers.iter().all(|vs| vs.contains(&*v))) + .filter(|v| version_specifiers.iter().all(|vs| (vs).contains(&*v))) .collect_vec(); if !valid.is_empty() { + let mut result = valid[0].clone(); + // Is there at least one version specifier that has PATCH digit? + let patch_vs = version_specifiers + .iter() + .any(|vs| vs.version().release().get(2).is_some()); + if select_latest { - return Ok(valid[0].clone()); + return Ok(result.clone()); } // Usually INSTANCE_PYTHON_VERSION @@ -183,9 +189,10 @@ impl PyV { // - We will iterate until find the closest version to target. // - If closest version has the same MINOR version, use it. // - If it differs in MINOR version, take latest PATCH version. - // - let mut result = None; + let [major, minor, ..] = result.release() else { + return Err(Error::InternalErr(format!("Failed to parse \"{}\". Available python versions are supposed to be in SEMVER format (MAJOR.MINOR)", *result))); + }; // This represents newest version with oldest MINOR: // // I Iterable Newest in MINOR @@ -195,12 +202,9 @@ impl PyV { // 4. 3.10.2 -> 3.10.2 // 5. 3.10.1 -> 3.10.2 // 6. 3.10.0 -> 3.10.2 - let mut newest_in_minor = None; - for v in valid.iter() { - if result.is_none() { - result.replace(v); - } + let mut newest_in_minor = (result.clone(), (*major, *minor)); + for v in valid.iter() { if v < &gv { // We will not continue if we start looking into versions older than gravity version. break; @@ -212,37 +216,46 @@ impl PyV { // Since we go top to down we can assume // the first occurence of new minor version contains the latest patch version. - if matches!(newest_in_minor, Some((_, mm)) if mm != (major, minor)) - || newest_in_minor.is_none() - { - newest_in_minor = Some((v.clone(), (major, minor))); + if newest_in_minor.1 != (*major, *minor) { + newest_in_minor = (v.clone(), (*major, *minor)); } if gravity_matcher.contains(v) { // return as soon as gravity matcher has first hit. - return Ok(v.clone()); + // Only in case version specifiers do specify PATCH version OR gravity version specify PATCH + if patch_vs || gv.release().get(2).is_some() { + return Ok(v.clone()); + } else { + let Some(release_numbers) = v.release().get(0..=1) else { + return Err(Error::InternalErr(format!( + "Failed to get release numbers from: \"{}\". ", + **v + ))); + }; + return Ok(PyV(pep440_rs::Version::new(release_numbers))); + } } // If we are still in the loop, it means that we are getting closer to gravity version else { - result = Some(v); + result = v.clone(); } } let [gravity_major, gravity_minor, ..] = gv.release() else { - return Err(Error::internal_err(format!("Cannot get MAJOR or/and MINOR version of python gravity version ({}). Something might be wrong with INSTANCE_PYTHON_VERSION.", &*gv))); + return Err(Error::internal_err(format!("Cannot get MAJOR nor MINOR version of python gravity version ({}). Something might be wrong with INSTANCE_PYTHON_VERSION.", &*gv))); }; - if let Some((v, mm)) = newest_in_minor { - if (gravity_major, gravity_minor) != mm { - return Ok(v); + if (*gravity_major, *gravity_minor) != newest_in_minor.1 { + // Return full version only if there is PATCH versions in version specifiers + if patch_vs { + return Ok(newest_in_minor.0); + } else { + let mm = newest_in_minor.1; + return Ok(PyV(pep440_rs::Version::new([mm.0, mm.1]))); } } - result - .ok_or(Error::internal_err( - "No python candidates found. This is a bug!", - )) - .map(ToOwned::to_owned) + Ok(result) } else { Err(anyhow!( " @@ -265,14 +278,24 @@ impl PyV { .into()) } } - /// e.g.: `/tmp/windmill/cache/python_3xy` - pub(crate) fn to_cache_dir(&self) -> String { + /// e.g.: `/tmp/windmill/cache/python_3_x_y` + pub(crate) fn to_cache_dir(&self, ignore_patch: bool) -> String { use windmill_common::worker::ROOT_CACHE_DIR; - format!("{ROOT_CACHE_DIR}{}", self.to_cache_dir_top_level()) + format!( + "{ROOT_CACHE_DIR}{}", + self.to_cache_dir_top_level(ignore_patch) + ) } /// e.g.: `python_3_x_y` - pub fn to_cache_dir_top_level(&self) -> String { + pub fn to_cache_dir_top_level(&self, ignore_patch: bool) -> String { + if ignore_patch { + if let [major, minor, ..] = self.release() { + return format!("python_{major}_{minor}"); + } + + tracing::warn!("failed to parse python's ({}) top level directory with no patch digit, fallback to full version.", self.to_string()); + } format!("python_{}", self.to_string().replace(".", "_")) } @@ -283,6 +306,7 @@ impl PyV { ) -> Self { let mut err = None; let pyv = match INSTANCE_PYTHON_VERSION.read().await.clone() { + Some(v) if &v == "default" => PyVAlias::default().into(), Some(v) => pep440_rs::Version::from_str(&v).unwrap_or_else(|_| { let v = PyVAlias::default().into(); err = Some(format!("\nCannot parse INSTANCE_PYTHON_VERSION ({:?}), fallback to latest_stable ({v:?})", *INSTANCE_PYTHON_VERSION)); @@ -575,7 +599,7 @@ impl PyV { // For the default version directory created during startup (main.rs) DirBuilder::new() .recursive(true) - .create(self.to_cache_dir()) + .create(self.to_cache_dir(false)) .await .expect("could not create initial worker dir"); @@ -762,7 +786,7 @@ mod tests { pyv("0.9.3"), pyv("0.9.2"), ], - pyv("0.9.4"), // + pyv("0.9"), // ) .await; } @@ -785,7 +809,7 @@ mod tests { pyv("0.8.1"), pyv("0.8.0"), ], - pyv("1.0.2"), // + pyv("1.0"), // ) .await; } @@ -824,7 +848,7 @@ mod tests { pyv("2.2.1"), pyv("2.2.0"), ], - pyv("2.2.2"), + pyv("2.2"), ) .await; } @@ -845,4 +869,151 @@ mod tests { ) .await; } + #[tokio::test] + async fn test_python_resolution_8() { + assert_resolution( + "2.2", + false, + vec![">2.2", ">=2.4", "<2.4.1"], + vec![ + pyv("2.4.1"), + pyv("2.4.0"), + pyv("2.3.1"), + pyv("2.3.0"), + pyv("2.2.1"), + pyv("2.2.0"), + pyv("2.1.1"), + pyv("2.1.0"), + ], + pyv("2.4.0"), + ) + .await; + } + #[tokio::test] + async fn test_python_resolution_9() { + assert_resolution( + "2.2", + true, + vec![">2.2", ">2.3", "<2.4.1"], + vec![ + pyv("2.4.1"), + pyv("2.4.0"), + pyv("2.3.1"), + pyv("2.3.0"), + pyv("2.2.1"), + pyv("2.2.0"), + pyv("2.1.1"), + pyv("2.1.0"), + ], + pyv("2.4.0"), + ) + .await; + } + #[tokio::test] + async fn test_python_resolution_10() { + assert_resolution( + "2.2", + false, + vec![">2.2", ">=2.4"], + vec![ + pyv("2.4.1"), + pyv("2.4.0"), + pyv("2.3.1"), + pyv("2.3.0"), + pyv("2.2.1"), + pyv("2.2.0"), + pyv("2.1.1"), + pyv("2.1.0"), + ], + // vec![pyv("2.4.1"), pyv("2.3"), pyv("2.2"), pyv("2.1")], + pyv("2.4"), + ) + .await; + } + #[tokio::test] + async fn test_python_resolution_11() { + assert_resolution( + "2.4.1", + false, + vec![">2.2", ">=2.3", "<2.4"], + vec![ + pyv("2.4.1"), + pyv("2.4.0"), + pyv("2.3.1"), + pyv("2.3.0"), + pyv("2.2.1"), + pyv("2.2.0"), + pyv("2.1.1"), + pyv("2.1.0"), + ], + // vec![pyv("2.4.1"), pyv("2.3"), pyv("2.2"), pyv("2.1")], + pyv("2.3"), + ) + .await; + } + #[tokio::test] + async fn test_python_resolution_12() { + assert_resolution( + "2.3", + false, + vec![">2.2", ">=2.3", "<2.4"], + vec![ + pyv("2.4.1"), + pyv("2.4.0"), + pyv("2.3.1"), + pyv("2.3.0"), + pyv("2.2.1"), + pyv("2.2.0"), + pyv("2.1.1"), + pyv("2.1.0"), + ], + // vec![pyv("2.4.1"), pyv("2.3"), pyv("2.2"), pyv("2.1")], + pyv("2.3"), + ) + .await; + } + #[tokio::test] + async fn test_python_resolution_13() { + assert_resolution( + "2.3.1", + false, + vec![">2.2", ">=2.3", "<2.4"], + vec![ + pyv("2.4.1"), + pyv("2.4.0"), + pyv("2.3.1"), + pyv("2.3.0"), + pyv("2.2.1"), + pyv("2.2.0"), + pyv("2.1.1"), + pyv("2.1.0"), + ], + // vec![pyv("2.4.1"), pyv("2.3"), pyv("2.2"), pyv("2.1")], + pyv("2.3.1"), + ) + .await; + } + #[tokio::test] + async fn test_python_resolution_14() { + assert_resolution( + "2.3.0", + false, + vec![], + vec![pyv("2.4.1"), pyv("2.4.0"), pyv("2.3.1")], + pyv("2.3.1"), + ) + .await; + } + + #[tokio::test] + async fn test_python_resolution_16() { + assert_resolution( + "2.3", + false, + vec![], + vec![pyv("2.4.1"), pyv("2.4.0"), pyv("2.3.1")], + pyv("2.3"), + ) + .await; + } } diff --git a/backend/windmill-worker/src/result_processor.rs b/backend/windmill-worker/src/result_processor.rs index 342c53a301..5056851389 100644 --- a/backend/windmill-worker/src/result_processor.rs +++ b/backend/windmill-worker/src/result_processor.rs @@ -12,7 +12,7 @@ use std::{ }; use tracing::{field, Instrument}; #[cfg(not(feature = "otel"))] -use windmill_common::otel_ee::FutureExt; +use windmill_common::otel_oss::FutureExt; use uuid::Uuid; @@ -32,7 +32,7 @@ use windmill_queue::{ append_logs, get_queued_job, CanceledBy, JobCompleted, MiniPulledJob, WrappedError, }; -use serde_json::{json, value::RawValue}; +use serde_json::{json, value::RawValue, Value}; use tokio::task::JoinHandle; @@ -41,10 +41,10 @@ use windmill_queue::{add_completed_job, add_completed_job_error}; use crate::{ bash_executor::ANSI_ESCAPE_RE, common::{error_to_value, read_result, save_in_cache}, - otel_ee::add_root_flow_job_to_otlp, + otel_oss::add_root_flow_job_to_otlp, worker_flow::update_flow_status_after_job_completion, JobCompletedReceiver, JobCompletedSender, SameWorkerSender, SendResult, UpdateFlow, - INIT_SCRIPT_TAG, + INIT_SCRIPT_TAG, SAME_WORKER_REQUIREMENTS, }; use windmill_common::client::AuthedClient; @@ -54,7 +54,7 @@ async fn process_jc( base_internal_url: &str, db: &DB, worker_dir: &str, - same_worker_tx: &SameWorkerSender, + same_worker_tx: Option<&SameWorkerSender>, job_completed_sender: &JobCompletedSender, #[cfg(feature = "benchmark")] bench: &mut BenchmarkIter, ) { @@ -76,7 +76,7 @@ async fn process_jc( } else { jc.job.id }; - windmill_common::otel_ee::set_span_parent(&span, &rj); + windmill_common::otel_oss::set_span_parent(&span, &rj); if let Some(lg) = jc.job.script_lang.as_ref() { span.record("language", lg.as_str()); @@ -104,8 +104,8 @@ async fn process_jc( jc, &base_internal_url, &db, - &worker_dir, - &same_worker_tx, + worker_dir, + same_worker_tx, &worker_name, job_completed_sender.clone(), #[cfg(feature = "benchmark")] @@ -119,6 +119,11 @@ async fn process_jc( } } +enum JobCompletedRx { + JobCompleted(SendResult), + Killpill, +} + pub fn start_background_processor( job_completed_rx: JobCompletedReceiver, job_completed_sender: JobCompletedSender, @@ -140,10 +145,6 @@ pub fn start_background_processor( #[cfg(feature = "benchmark")] let mut infos = BenchmarkInfo::new(); - enum JobCompletedRx { - JobCompleted(SendResult), - Killpill, - } //if we have been killed, we want to drain the queue of jobs while let Some(sr) = { if has_been_killed && same_worker_queue_size.load(Ordering::SeqCst) == 0 { @@ -186,7 +187,7 @@ pub fn start_background_processor( &base_internal_url, &db, &worker_dir, - &same_worker_tx, + Some(&same_worker_tx), &job_completed_sender, #[cfg(feature = "benchmark")] &mut bench, @@ -229,19 +230,19 @@ pub fn start_background_processor( tracing::info!(parent_flow = %flow, "updating flow status"); if let Err(e) = update_flow_status_after_job_completion( &db, - &AuthedClient { - base_internal_url: base_internal_url.to_string(), - workspace: w_id.clone(), - token: token.clone(), - force_client: None, - }, + &AuthedClient::new( + base_internal_url.to_string(), + w_id.clone(), + token.clone(), + None, + ), flow, &Uuid::nil(), &w_id, success, Arc::new(result), true, - same_worker_tx.clone(), + &same_worker_tx, &worker_dir, stop_early_override, &worker_name, @@ -276,7 +277,7 @@ pub fn start_background_processor( async fn send_job_completed(job_completed_tx: JobCompletedSender, jc: JobCompleted) { job_completed_tx .send_job(jc, true) - .with_context(windmill_common::otel_ee::otel_ctx()) + .with_context(windmill_common::otel_oss::otel_ctx()) .await .expect("send job completed") } @@ -312,7 +313,7 @@ pub async fn process_result( duration, }, ) - .with_context(windmill_common::otel_ee::otel_ctx()) + .with_context(windmill_common::otel_oss::otel_ctx()) .await; Ok(true) } @@ -374,7 +375,7 @@ pub async fn process_result( duration, }, ) - .with_context(windmill_common::otel_ee::otel_ctx()) + .with_context(windmill_common::otel_oss::otel_ctx()) .await; Ok(false) } @@ -386,23 +387,19 @@ pub async fn handle_receive_completed_job( base_internal_url: &str, db: &DB, worker_dir: &str, - same_worker_tx: &SameWorkerSender, + same_worker_tx: Option<&SameWorkerSender>, worker_name: &str, job_completed_tx: JobCompletedSender, #[cfg(feature = "benchmark")] bench: &mut BenchmarkIter, ) -> Option> { let token = jc.token.clone(); let workspace = jc.job.workspace_id.clone(); - let client = AuthedClient { - base_internal_url: base_internal_url.to_string(), - workspace, - token, - force_client: None, - }; + let client = AuthedClient::new(base_internal_url.to_string(), workspace, token, None); let job = jc.job.clone(); let mem_peak = jc.mem_peak.clone(); let canceled_by = jc.canceled_by.clone(); - match process_completed_job( + + let processed_completed_job = process_completed_job( jc, &client, db, @@ -413,8 +410,9 @@ pub async fn handle_receive_completed_job( #[cfg(feature = "benchmark")] bench, ) - .await - { + .await; + + match processed_completed_job { Err(err) => { handle_job_error( db, @@ -454,7 +452,7 @@ pub async fn process_completed_job( client: &AuthedClient, db: &DB, worker_dir: &str, - same_worker_tx: SameWorkerSender, + same_worker_tx: Option<&SameWorkerSender>, worker_name: &str, job_completed_tx: JobCompletedSender, #[cfg(feature = "benchmark")] bench: &mut BenchmarkIter, @@ -530,7 +528,7 @@ pub async fn process_completed_job( true, result, false, - same_worker_tx.clone(), + &same_worker_tx.expect(SAME_WORKER_REQUIREMENTS).to_owned(), &worker_dir, None, worker_name, @@ -570,7 +568,7 @@ pub async fn process_completed_job( false, Arc::new(serde_json::value::to_raw_value(&result).unwrap()), false, - same_worker_tx, + &same_worker_tx.expect(SAME_WORKER_REQUIREMENTS).to_owned(), &worker_dir, None, worker_name, @@ -587,6 +585,34 @@ pub async fn process_completed_job( return Ok(None); } +async fn handle_non_flow_job_error( + db: &DB, + job: &MiniPulledJob, + mem_peak: i32, + canceled_by: Option, + err: Value, + worker_name: &str, +) -> Result { + append_logs( + &job.id, + &job.workspace_id, + format!("Unexpected error during job execution:\n{err:#?}"), + &db.into(), + ) + .await; + add_completed_job_error( + db, + job, + mem_peak, + canceled_by, + err, + worker_name, + false, + None, + ) + .await +} + #[tracing::instrument(name = "job_error", level = "info", skip_all, fields(job_id = %job.id))] pub async fn handle_job_error( db: &DB, @@ -596,7 +622,7 @@ pub async fn handle_job_error( canceled_by: Option, err: Error, unrecoverable: bool, - same_worker_tx: SameWorkerSender, + same_worker_tx: Option<&SameWorkerSender>, worker_dir: &str, worker_name: &str, job_completed_tx: JobCompletedSender, @@ -605,22 +631,13 @@ pub async fn handle_job_error( let err = error_to_value(err); let update_job_future = || async { - append_logs( - &job.id, - &job.workspace_id, - format!("Unexpected error during job execution:\n{err:#?}"), - &db.into(), - ) - .await; - add_completed_job_error( + handle_non_flow_job_error( db, job, mem_peak, canceled_by.clone(), err.clone(), worker_name, - false, - None, ) .await }; @@ -649,7 +666,7 @@ pub async fn handle_job_error( false, Arc::new(serde_json::value::to_raw_value(&wrapped_error).unwrap()), unrecoverable, - same_worker_tx, + &same_worker_tx.expect(SAME_WORKER_REQUIREMENTS).clone(), worker_dir, None, worker_name, diff --git a/backend/windmill-worker/src/rust_executor.rs b/backend/windmill-worker/src/rust_executor.rs index 0255783f6f..045158511e 100644 --- a/backend/windmill-worker/src/rust_executor.rs +++ b/backend/windmill-worker/src/rust_executor.rs @@ -66,6 +66,7 @@ mount { #[cfg(not(debug_assertions))] const DEV_CONF_NSJAIL: &'static str = ""; +#[cfg(not(windows))] lazy_static::lazy_static! { static ref CARGO_HOME_DEFAULT: String = format!("{}/.cargo", *HOME_DIR); static ref RUSTUP_HOME_DEFAULT: String = format!("{}/.rustup", *HOME_DIR); diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index e15c24ef70..56ac7c00e0 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -18,23 +18,23 @@ use windmill_common::{ cache::{future::FutureCachedExt, ScriptData, ScriptMetadata}, schema::{should_validate_schema, SchemaValidator}, scripts::PREVIEW_IS_TAR_CODEBASE_HASH, - utils::WarnAfterExt, + utils::{create_directory_async, WarnAfterExt}, worker::{ - write_file, Connection, HttpClient, MAX_TIMEOUT, ROOT_CACHE_DIR, ROOT_CACHE_NOMOUNT_DIR, - TMP_DIR, + make_pull_query, write_file, Connection, HttpClient, MAX_TIMEOUT, ROOT_CACHE_DIR, + ROOT_CACHE_NOMOUNT_DIR, TMP_DIR, }, KillpillSender, }; #[cfg(feature = "enterprise")] -use windmill_common::ee::LICENSE_KEY_VALID; +use windmill_common::ee_oss::LICENSE_KEY_VALID; use anyhow::Result; use const_format::concatcp; #[cfg(feature = "prometheus")] use prometheus::IntCounter; -use tracing::{field, Instrument}; +use tracing::{field, Instrument, Span}; #[cfg(feature = "prometheus")] use windmill_common::METRICS_DEBUG_ENABLED; #[cfg(feature = "prometheus")] @@ -44,7 +44,7 @@ use serde::{Deserialize, Serialize}; use sqlx::types::Json; use std::{ collections::HashMap, - fs::DirBuilder, + fmt::Display, sync::{ atomic::{AtomicBool, AtomicU16, Ordering}, Arc, @@ -168,15 +168,10 @@ use windmill_common::bench::{benchmark_init, BenchmarkInfo, BenchmarkIter}; use windmill_common::add_time; -pub const PY310_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "python_310"); -pub const PY311_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "python_311"); -pub const PY312_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "python_312"); -pub const PY313_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "python_313"); - -pub const TAR_PY310_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "tar/python_310"); -pub const TAR_PY311_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "tar/python_311"); -pub const TAR_PY312_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "tar/python_312"); -pub const TAR_PY313_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "tar/python_313"); +pub const PY310_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "python_3_10"); +pub const PY311_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "python_3_11"); +pub const PY312_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "python_3_12"); +pub const PY313_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "python_3_13"); pub const TAR_JAVA_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "tar/java"); @@ -210,6 +205,9 @@ const NUM_SECS_READINGS: u64 = 60; const INCLUDE_DEPS_PY_SH_CONTENT: &str = include_str!("../nsjail/download_deps.py.sh"); +const WORKER_SHELL_NAP_TIME_DURATION: u64 = 15; +const TIMEOUT_TO_RESET_WORKER_SHELL_NAP_TIME_DURATION: u64 = 2 * 60; + pub const DEFAULT_SLEEP_QUEUE: u64 = 50; // only 1 native job so that we don't have to worry about concurrency issues on non dedicated native jobs workers @@ -247,6 +245,8 @@ lazy_static::lazy_static! { const DOTNET_DEFAULT_PATH: &str = "C:\\Program Files\\dotnet\\dotnet.exe"; #[cfg(unix)] const DOTNET_DEFAULT_PATH: &str = "/usr/bin/dotnet"; +pub const SAME_WORKER_REQUIREMENTS: &'static str = + "SameWorkerSender is required because this job may be part of a flow"; lazy_static::lazy_static! { @@ -391,6 +391,31 @@ lazy_static::lazy_static! { pub static ref WIN_ENVS: Envs = vec![]; } +#[derive(Debug)] +pub enum NextJob { + Sql(PulledJob), + Http(JobAndPerms), +} + +impl NextJob { + pub fn job(self) -> MiniPulledJob { + match self { + NextJob::Sql(job) => job.job, + NextJob::Http(job) => job.job, + } + } +} + +impl std::ops::Deref for NextJob { + type Target = MiniPulledJob; + fn deref(&self) -> &Self::Target { + match self { + NextJob::Sql(job) => &job.job, + NextJob::Http(job) => &job.job, + } + } +} + //only matter if CLOUD_HOSTED pub const MAX_RESULT_SIZE: usize = 1024 * 1024 * 2; // 2MB @@ -431,20 +456,26 @@ impl JobCompletedReceiver { } impl JobCompletedSender { + pub fn new_job_completed_sender_sql(buffer_size: u8) -> (Self, JobCompletedReceiver) { + let (sender, receiver) = flume::bounded::(buffer_size as usize); + let (unbounded_sender, unbounded_rx) = flume::unbounded::(); + let (killpill_tx, killpill_rx) = broadcast::channel::<()>(10); + ( + Self::Sql(SqlJobCompletedSender { sender, unbounded_sender, killpill_tx }), + JobCompletedReceiver { bounded_rx: receiver, killpill_rx, unbounded_rx }, + ) + } + pub fn new(conn: &Connection, buffer_size: u8) -> (Self, Option) { match conn { Connection::Sql(_) => { - let (sender, receiver) = flume::bounded::(buffer_size as usize); - let (unbounded_sender, unbounded_rx) = flume::unbounded::(); - let (killpill_tx, killpill_rx) = broadcast::channel::<()>(10); - ( - Self::Sql(SqlJobCompletedSender { sender, unbounded_sender, killpill_tx }), - Some(JobCompletedReceiver { bounded_rx: receiver, killpill_rx, unbounded_rx }), - ) + let result = Self::new_job_completed_sender_sql(buffer_size); + (result.0, Some(result.1)) } Connection::Http(client) => (Self::Http(client.clone()), None), } } + pub fn new_never_used() -> (Self, Option>) { (Self::NeverUsed, None) } @@ -622,6 +653,233 @@ fn add_outstanding_wait_time( } } +async fn extract_job_and_perms(job: NextJob, conn: &Connection) -> JobAndPerms { + match (job, conn) { + (NextJob::Sql(job), Connection::Sql(db)) => job.get_job_and_perms(db).await, + (NextJob::Sql(_), Connection::Http(_)) => panic!("sql job on http connection"), + (NextJob::Http(job), _) => job, + } +} + +fn create_span(arc_job: &Arc, worker_name: &str, hostname: &str) -> Span { + let span = tracing::span!(tracing::Level::INFO, "job", + job_id = %arc_job.id, root_job = field::Empty, workspace_id = %arc_job.workspace_id, worker = %worker_name, hostname = %hostname, tag = %arc_job.tag, + language = field::Empty, + script_path = field::Empty, flow_step_id = field::Empty, parent_job = field::Empty, + otel.name = field::Empty); + + let rj = arc_job.flow_innermost_root_job.unwrap_or(arc_job.id); + + if let Some(lg) = arc_job.script_lang.as_ref() { + span.record("language", lg.as_str()); + } + if let Some(step_id) = arc_job.flow_step_id.as_ref() { + span.record("otel.name", format!("job {}", step_id).as_str()); + span.record("flow_step_id", step_id.as_str()); + } else { + span.record("otel.name", "job"); + } + if let Some(parent_job) = arc_job.parent_job.as_ref() { + span.record("parent_job", parent_job.to_string().as_str()); + } + if let Some(script_path) = arc_job.runnable_path.as_ref() { + span.record("script_path", script_path.as_str()); + } + if let Some(root_job) = arc_job.flow_innermost_root_job.as_ref() { + span.record("root_job", root_job.to_string().as_str()); + } + + windmill_common::otel_oss::set_span_parent(&span, &rj); + span +} + +pub async fn handle_all_job_kind_error( + conn: &Connection, + authed_client: &AuthedClient, + job: Arc, + err: Error, + same_worker_tx: Option<&SameWorkerSender>, + worker_dir: &str, + worker_name: &str, + job_completed_tx: JobCompletedSender, + #[cfg(feature = "benchmark")] bench: &mut BenchmarkIter, +) { + match conn { + Connection::Sql(db) => { + handle_job_error( + db, + authed_client, + job.as_ref(), + 0, + None, + err, + false, + same_worker_tx, + &worker_dir, + &worker_name, + job_completed_tx.clone(), + #[cfg(feature = "benchmark")] + bench, + ) + .await; + } + Connection::Http(_) => { + job_completed_tx + .send_job( + JobCompleted { + preprocessed_args: None, + job: job.clone(), + result: Arc::new(windmill_common::worker::to_raw_value(&error_to_value( + err, + ))), + result_columns: None, + mem_peak: 0, + canceled_by: None, + success: false, + cached_res_path: None, + token: authed_client.token.clone(), + duration: None, + }, + false, + ) + .await + .expect("send job completed"); + } + } +} + +pub fn start_interactive_worker_shell( + conn: Connection, + hostname: String, + worker_name: String, + mut killpill_rx: tokio::sync::broadcast::Receiver<()>, + job_completed_tx: JobCompletedSender, + base_internal_url: String, + worker_dir: String, +) -> JoinHandle<()> { + tokio::spawn(async move { + let mut occupancy_metrics = OccupancyMetrics::new(Instant::now()); + + let mut last_executed_job: Option = + Instant::now().checked_sub(Duration::from_millis(2500)); + + loop { + if let Ok(_) = killpill_rx.try_recv() { + break; + } else { + let pulled_job = match &conn { + Connection::Sql(db) => { + let query = ("".to_string(), make_pull_query(&[hostname.to_owned()])); + + #[cfg(feature = "benchmark")] + let mut bench = windmill_common::bench::BenchmarkIter::new(); + let job = pull( + &db, + false, + &worker_name, + Some(&query), + #[cfg(feature = "benchmark")] + &mut bench, + ) + .await; + + job.map(|x| x.job.map(NextJob::Sql)) + } + Connection::Http(client) => { + crate::agent_workers::pull_job(&client, None, Some(true)) + .await + .map_err(|e| error::Error::InternalErr(e.to_string())) + .map(|x| x.map(|y| NextJob::Http(y))) + } + }; + + match pulled_job { + Ok(Some(job)) => { + tracing::debug!(worker = %worker_name, hostname = %hostname, "started handling of job {}", job.id); + + let job_dir = create_job_dir(&worker_dir, job.id).await; + #[cfg(feature = "benchmark")] + let mut bench = windmill_common::bench::BenchmarkIter::new(); + + let JobAndPerms { + job, + raw_code, + raw_lock, + raw_flow, + parent_runnable_path, + token, + precomputed_agent_info: precomputed_bundle, + } = extract_job_and_perms(job, &conn).await; + + let authed_client = AuthedClient::new( + base_internal_url.to_owned(), + job.workspace_id.clone(), + token, + None, + ); + + let arc_job = Arc::new(job); + + let _ = handle_queued_job( + arc_job.clone(), + raw_code, + raw_lock, + raw_flow, + parent_runnable_path, + &conn, + &authed_client, + &hostname, + &worker_name, + &worker_dir, + &job_dir, + None, + &base_internal_url, + job_completed_tx.clone(), + &mut occupancy_metrics, + &mut killpill_rx, + precomputed_bundle, + #[cfg(feature = "benchmark")] + &mut bench, + ) + .await; + + last_executed_job = Some(Instant::now()); + } + Ok(None) => { + let now = Instant::now(); + match last_executed_job { + Some(last) + if now.duration_since(last).as_secs() + > TIMEOUT_TO_RESET_WORKER_SHELL_NAP_TIME_DURATION => + { + tokio::time::sleep(Duration::from_secs( + WORKER_SHELL_NAP_TIME_DURATION, + )) + .await; + } + _ => { + tokio::time::sleep(Duration::from_millis(*SLEEP_QUEUE)).await; + } + } + } + + Err(err) => { + tracing::error!(worker = %worker_name, hostname = %hostname, "Failed to pull jobs: {}", err); + } + }; + } + } + }) +} + +async fn create_job_dir(worker_directory: &str, job_id: impl Display) -> String { + let job_dir_path = format!("{}/{}", worker_directory, job_id); + + create_directory_async(&job_dir_path).await; + + job_dir_path +} + pub async fn run_worker( conn: &Connection, hostname: &str, @@ -686,10 +944,7 @@ pub async fn run_worker( write_file(&HOME_ENV, ".netrc", netrc).expect("could not write netrc"); } - DirBuilder::new() - .recursive(true) - .create(&worker_dir) - .expect("could not create initial worker dir"); + create_directory_async(&worker_dir).await; if !*DISABLE_NSJAIL { let _ = write_file( @@ -948,6 +1203,25 @@ pub async fn run_worker( )), _ => None, }; + // If we're the first worker to run, we start another background process that listens for a specific tag. + // This tag is associated only with jobs using Bash as the script language. + // For agent workers, the expected tag format is the worker name suffixed with "-ssh". + // For regular workers, the tag is simply the machine's hostname and if not found the randomly generated hostname. + let interactive_shell = if i_worker == 1 { + let it_shell = start_interactive_worker_shell( + conn.clone(), + hostname.to_owned(), + worker_name.clone(), + killpill_rx.resubscribe(), + job_completed_tx.clone(), + base_internal_url.to_owned(), + worker_dir.clone(), + ); + + Some(it_shell) + } else { + None + }; let mut last_executed_job: Option = None; @@ -1058,7 +1332,7 @@ pub async fn run_worker( if !valid_key { tracing::error!( worker = %worker_name, hostname = %hostname, - "Invalid license key, workers require a valid license key, sleeping for 30s waiting for valid key to be set" + "Invalid license key, workers require a valid license key, sleeping for 10s waiting for valid key to be set" ); tokio::time::sleep(Duration::from_secs(10)).await; continue; @@ -1128,29 +1402,6 @@ pub async fn run_worker( } else { tracing::info!("benchmark not finished, still pulling jobs {}", infos.iters); } - enum NextJob { - Sql(PulledJob), - Http(JobAndPerms), - } - - impl NextJob { - pub fn job(self) -> MiniPulledJob { - match self { - NextJob::Sql(job) => job.job, - NextJob::Http(job) => job.job, - } - } - } - - impl std::ops::Deref for NextJob { - type Target = MiniPulledJob; - fn deref(&self) -> &Self::Target { - match self { - NextJob::Sql(job) => &job.job, - NextJob::Http(job) => &job.job, - } - } - } let next_job = { // println!("2: {:?}", instant.elapsed()); @@ -1191,6 +1442,7 @@ pub async fn run_worker( "/api/agent_workers/same_worker_job/{}", same_worker_job.job_id ), + None, &same_worker_job, ) .await @@ -1296,7 +1548,7 @@ pub async fn run_worker( } job.map(|x| x.job.map(NextJob::Sql)) } - Connection::Http(client) => crate::agent_workers::pull_job(&client) + Connection::Http(client) => crate::agent_workers::pull_job(&client, None, None) .await .map_err(|e| error::Error::InternalErr(e.to_string())) .map(|x| x.map(|y| NextJob::Http(y))), @@ -1344,6 +1596,7 @@ pub async fn run_worker( } } } + if matches!(job.kind, JobKind::Noop) { add_time!(bench, "send job completed START"); job_completed_tx @@ -1425,20 +1678,12 @@ pub async fn run_worker( // fields macro we can make a job id that only appears when // the job is defined? - let job_dir = format!("{worker_dir}/{}", job.id); - - DirBuilder::new() - .recursive(true) - .create(&job_dir) - .expect("could not create job dir"); + let job_dir = create_job_dir(&worker_dir, job.id).await; let same_worker = job.same_worker; let folder = if job.script_lang == Some(ScriptLang::Go) { - DirBuilder::new() - .recursive(true) - .create(&format!("{job_dir}/go")) - .expect("could not create go dir"); + create_directory_async(&format!("{job_dir}/go")).await; "/go" } else { "" @@ -1450,26 +1695,22 @@ pub async fn run_worker( if tokio::fs::metadata(target).await.is_err() { let parent_flow = job.parent_job.unwrap(); let parent_shared_dir = format!("{worker_dir}/{parent_flow}/shared"); - DirBuilder::new() - .recursive(true) - .create(&parent_shared_dir) - .expect("could not create parent shared dir"); - + create_directory_async(&parent_shared_dir).await; symlink(&parent_shared_dir, target) .await .expect("could not symlink target"); } } else { - DirBuilder::new() - .recursive(true) - .create(target) - .expect("could not create shared dir"); + create_directory_async(target).await; } #[cfg(feature = "prometheus")] let tag = job.tag.clone(); let is_init_script: bool = job.tag.as_str() == INIT_SCRIPT_TAG; + let is_flow = job.is_flow(); + let job_id = job.id; + let JobAndPerms { job, raw_code, @@ -1478,70 +1719,32 @@ pub async fn run_worker( parent_runnable_path, token, precomputed_agent_info: precomputed_bundle, - } = match (job, &conn) { - (NextJob::Sql(job), Connection::Sql(db)) => job.get_job_and_perms(db).await, - (NextJob::Sql(_), Connection::Http(_)) => { - panic!("sql job on http connection") - } - (NextJob::Http(job), _) => job, - }; + } = extract_job_and_perms(job, &conn).await; - // let token = create_token(&db, &job, job_perms).await; - let authed_client = AuthedClient { - base_internal_url: base_internal_url.to_string(), + let authed_client = AuthedClient::new( + base_internal_url.to_owned(), + job.workspace_id.clone(), token, - workspace: job.workspace_id.to_string(), - force_client: None, - }; + None, + ); let arc_job = Arc::new(job); - add_time!(bench, "handle_queued_job START"); - let span = tracing::span!(tracing::Level::INFO, "job", - job_id = %arc_job.id, root_job = field::Empty, workspace_id = %arc_job.workspace_id, worker = %worker_name, hostname = %hostname, tag = %arc_job.tag, - language = field::Empty, - script_path = field::Empty, flow_step_id = field::Empty, parent_job = field::Empty, - otel.name = field::Empty); - let rj = if let Some(root_job) = arc_job.flow_innermost_root_job { - root_job - } else { - arc_job.id - }; - if let Some(lg) = arc_job.script_lang.as_ref() { - span.record("language", lg.as_str()); - } - if let Some(step_id) = arc_job.flow_step_id.as_ref() { - span.record("otel.name", format!("job {}", step_id).as_str()); - span.record("flow_step_id", step_id.as_str()); - } else { - span.record("otel.name", "job"); - } - if let Some(parent_job) = arc_job.parent_job.as_ref() { - span.record("parent_job", parent_job.to_string().as_str()); - } - if let Some(script_path) = arc_job.runnable_path.as_ref() { - span.record("script_path", script_path.as_str()); - } - if let Some(root_job) = arc_job.flow_innermost_root_job.as_ref() { - span.record("root_job", root_job.to_string().as_str()); - } + let span = create_span(&arc_job, &worker_name, hostname); - windmill_common::otel_ee::set_span_parent(&span, &rj); - // span.context().span().add_event_with_timestamp("job created".to_string(), arc_job.created_at.into(), vec![]); - - match handle_queued_job( + let job_result = handle_queued_job( arc_job.clone(), raw_code, raw_lock, raw_flow, parent_runnable_path, - conn, + &conn, &authed_client, - &hostname, + hostname, &worker_name, &worker_dir, &job_dir, - same_worker_tx.clone(), + Some(same_worker_tx.clone()), base_internal_url, job_completed_tx.clone(), &mut occupancy_metrics, @@ -1551,74 +1754,40 @@ pub async fn run_worker( &mut bench, ) .instrument(span) - .await - { + .await; + + match job_result { + Ok(false) if is_init_script => { + tracing::error!("init script job failed, exiting"); + update_worker_ping_for_failed_init_script(conn, &worker_name, job_id) + .await; + break; + } Err(err) => { - match conn { - Connection::Sql(db) => { - handle_job_error( - db, - &authed_client, - arc_job.as_ref(), - 0, - None, - err, - false, - same_worker_tx.clone(), - &worker_dir, - &worker_name, - job_completed_tx.clone(), - #[cfg(feature = "benchmark")] - &mut bench, - ) - .await; - } - Connection::Http(_) => { - job_completed_tx - .send_job( - JobCompleted { - preprocessed_args: None, - job: arc_job.clone(), - result: Arc::new( - windmill_common::worker::to_raw_value( - &error_to_value(err), - ), - ), - result_columns: None, - mem_peak: 0, - canceled_by: None, - success: false, - cached_res_path: None, - token: authed_client.token.clone(), - duration: None, - }, - false, - ) - .await - .expect("send job completed"); - } - } + handle_all_job_kind_error( + &conn, + &authed_client, + arc_job.clone(), + err, + Some(&same_worker_tx), + &worker_dir, + &worker_name, + job_completed_tx.clone(), + #[cfg(feature = "benchmark")] + &mut bench, + ) + .await; if is_init_script { tracing::error!("init script job failed (in handler), exiting"); update_worker_ping_for_failed_init_script( conn, &worker_name, - arc_job.id, + job_id, ) .await; break; } } - Ok(false) if is_init_script => { - tracing::error!("init script job failed, exiting"); - update_worker_ping_for_failed_init_script( - conn, - &worker_name, - arc_job.id, - ) - .await; - break; - } _ => {} } @@ -1643,8 +1812,7 @@ pub async fn run_worker( .await; } - if !KEEP_JOB_DIR.load(Ordering::Relaxed) && !(arc_job.is_flow() && same_worker) - { + if !KEEP_JOB_DIR.load(Ordering::Relaxed) && !(is_flow && same_worker) { let _ = tokio::fs::remove_dir_all(job_dir).await; } } @@ -1726,6 +1894,11 @@ pub async fn run_worker( tracing::error!("error in awaiting send_result process: {e:?}") } } + if let Some(interactive_shell) = interactive_shell { + if let Err(e) = interactive_shell.await { + tracing::error!("error in awaiting interactive_shell process: {e:?}") + } + } tracing::info!(worker = %worker_name, hostname = %hostname, "worker {} exited", worker_name); tracing::info!(worker = %worker_name, hostname = %hostname, "number of jobs executed: {}", jobs_executed); } @@ -1814,7 +1987,7 @@ pub struct PreviousResult<'a> { pub previous_result: Option<&'a RawValue>, } -async fn handle_queued_job( +pub async fn handle_queued_job( job: Arc, raw_code: Option, raw_lock: Option, @@ -1826,7 +1999,7 @@ async fn handle_queued_job( worker_name: &str, worker_dir: &str, job_dir: &str, - same_worker_tx: SameWorkerSender, + same_worker_tx: Option, base_internal_url: &str, job_completed_tx: JobCompletedSender, occupancy_metrics: &mut OccupancyMetrics, @@ -1939,7 +2112,7 @@ async fn handle_queued_job( .to_string(); append_logs(&job.id, &job.workspace_id, logs, conn).await; } - job_completed_tx + let result = job_completed_tx .send_job( JobCompleted { preprocessed_args: None, @@ -1955,8 +2128,16 @@ async fn handle_queued_job( }, true, ) - .await - .expect("send job completed"); + .await; + + match result { + Ok(_) => { + tracing::debug!("Send job completed") + } + Err(err) => { + tracing::error!("An error occurred while sending job completed: {:#?}", err) + } + } return Ok(true); } @@ -1975,7 +2156,7 @@ async fn handle_queued_job( db, &client, None, - same_worker_tx, + &same_worker_tx.expect(SAME_WORKER_REQUIREMENTS), worker_dir, job_completed_tx.clone(), worker_name, @@ -2144,7 +2325,6 @@ async fn handle_queued_job( { return Ok(false); } - process_result( job, result.map(|x| Arc::new(x)), diff --git a/backend/windmill-worker/src/worker_flow.rs b/backend/windmill-worker/src/worker_flow.rs index 6c0b133828..95ffc28062 100644 --- a/backend/windmill-worker/src/worker_flow.rs +++ b/backend/windmill-worker/src/worker_flow.rs @@ -66,7 +66,7 @@ use windmill_queue::{ type DB = sqlx::Pool; -use windmill_audit::audit_ee::{audit_log, AuditAuthor}; +use windmill_audit::audit_oss::{audit_log, AuditAuthor}; use windmill_audit::ActionKind; use windmill_queue::{canceled_job_to_result, push}; @@ -80,7 +80,7 @@ pub async fn update_flow_status_after_job_completion( success: bool, result: Arc>, unrecoverable: bool, - same_worker_tx: SameWorkerSender, + same_worker_tx: &SameWorkerSender, worker_dir: &str, stop_early_override: Option, worker_name: &str, @@ -89,6 +89,7 @@ pub async fn update_flow_status_after_job_completion( ) -> error::Result>> { // this is manual tailrecursion because async_recursion blows up the stack potentially_crash_for_testing(); + let mut rec = RecUpdateFlowStatusAfterJobCompletion { flow, job_id_for_status: job_id_for_status.clone(), @@ -109,7 +110,7 @@ pub async fn update_flow_status_after_job_completion( rec.success, rec.result, unrecoverable, - same_worker_tx.clone(), + same_worker_tx, worker_dir, rec.stop_early_override, rec.skip_error_handler, @@ -134,7 +135,7 @@ pub async fn update_flow_status_after_job_completion( error: json!(e.to_string()), }))), true, - same_worker_tx.clone(), + same_worker_tx, worker_dir, rec.stop_early_override, rec.skip_error_handler, @@ -147,6 +148,7 @@ pub async fn update_flow_status_after_job_completion( } }; unrecoverable = false; + match nrec { UpdateFlowStatusAfterJobCompletion::Done(job) => { add_time!(bench, "update flow status internal END"); @@ -226,7 +228,7 @@ pub async fn update_flow_status_after_job_completion_internal( mut success: bool, result: Arc>, unrecoverable: bool, - same_worker_tx: SameWorkerSender, + same_worker_tx: &SameWorkerSender, worker_dir: &str, stop_early_override: Option, skip_error_handler: bool, @@ -625,9 +627,10 @@ pub async fn update_flow_status_after_job_completion_internal( sqlx::query!( "UPDATE v2_job_queue q SET suspend = 0 FROM v2_job j, v2_job_status f - WHERE parent_job = $1 + WHERE q.workspace_id = $1 AND q.suspend = $3 AND j.parent_job = $2 AND f.id = j.id AND q.id = j.id - AND suspend = $2 AND (f.flow_status->'step')::int = 0", + AND (f.flow_status->'step')::int = 0", + w_id, flow, nindex ) @@ -1289,7 +1292,7 @@ pub async fn update_flow_status_after_job_completion_internal( db, client, Some(nresult.clone()), - same_worker_tx.clone(), + same_worker_tx, worker_dir, job_completed_tx, worker_name, @@ -1586,7 +1589,7 @@ pub async fn handle_flow( db: &sqlx::Pool, client: &AuthedClient, last_result: Option>>, - same_worker_tx: SameWorkerSender, + same_worker_tx: &SameWorkerSender, worker_dir: &str, job_completed_tx: JobCompletedSender, worker_name: &str, @@ -1643,7 +1646,7 @@ pub async fn handle_flow( db, client, last_result.clone(), - same_worker_tx.clone(), + same_worker_tx, worker_dir, worker_name, ) @@ -1736,7 +1739,7 @@ async fn push_next_flow_job( db: &sqlx::Pool, client: &AuthedClient, last_job_result: Option>>, - same_worker_tx: SameWorkerSender, + same_worker_tx: &SameWorkerSender, worker_dir: &str, worker_name: &str, ) -> error::Result { @@ -4006,7 +4009,7 @@ async fn flow_to_payload( w_id: &str, db: &DB, ) -> Result { - let FlowVersionInfo { version, on_behalf_of_email, edited_by, .. } = + let FlowVersionInfo { version, on_behalf_of_email, edited_by, tag, .. } = get_latest_flow_version_info_for_path(db, w_id, &path, true).await?; let on_behalf_of = if let Some(email) = on_behalf_of_email { Some(OnBehalfOf { email, permissioned_as: username_to_permissioned_as(&edited_by) }) @@ -4015,7 +4018,7 @@ async fn flow_to_payload( }; let payload = JobPayload::Flow { path, dedicated_worker: None, apply_preprocessor: false, version }; - Ok(JobPayloadWithTag { payload, tag: None, delete_after_use, timeout: None, on_behalf_of }) + Ok(JobPayloadWithTag { payload, tag, delete_after_use, timeout: None, on_behalf_of }) } async fn script_to_payload( diff --git a/backend/windmill-worker/src/worker_lockfiles.rs b/backend/windmill-worker/src/worker_lockfiles.rs index 6710f718c4..47bb07c438 100644 --- a/backend/windmill-worker/src/worker_lockfiles.rs +++ b/backend/windmill-worker/src/worker_lockfiles.rs @@ -1965,12 +1965,11 @@ async fn ansible_dep( use windmill_parser_yaml::add_versions_to_requirements_yaml; use crate::ansible_executor::{ - create_ansible_cfg, get_collection_locks, get_git_ssh_cmd, get_role_locks, - install_galaxy_collections, - }; + create_ansible_cfg, get_collection_locks, get_git_ssh_cmd, get_role_locks, + install_galaxy_collections, + }; use windmill_common::client::AuthedClient; - let python_lockfile = python_dep( reqs.python_reqs.join("\n").to_string(), job_id, @@ -1989,12 +1988,12 @@ async fn ansible_dep( let conn = &Connection::Sql(db.clone()); - let authed_client = AuthedClient { - base_internal_url: base_internal_url.to_string(), - token: token.to_string(), - workspace: w_id.to_string(), - force_client: None, - }; + let authed_client = AuthedClient::new( + base_internal_url.to_string(), + w_id.to_string(), + token.to_string(), + None, + ); let git_ssh_cmd = get_git_ssh_cmd(&reqs, job_dir, &authed_client).await?; @@ -2102,7 +2101,12 @@ async fn capture_dependency_job( ( job_raw_code.to_owned(), - crate::PyV::parse_from_requirements(&split_requirements(job_raw_code)), + match crate::PyV::try_parse_from_requirements(&split_requirements( + job_raw_code, + )) { + Some(pyv) => pyv, + None => crate::PyV::gravitational_version(job_id, w_id, None).await, + }, ) } else { let mut version_specifiers = vec![]; @@ -2189,11 +2193,6 @@ async fn capture_dependency_job( } } ScriptLang::Go => { - if raw_deps { - return Err(Error::ExecutionErr( - "Raw dependencies not supported for go".to_string(), - )); - } install_go_dependencies( job_id, job_raw_code, @@ -2204,6 +2203,7 @@ async fn capture_dependency_job( false, false, false, + raw_deps, worker_name, w_id, occupancy_metrics, diff --git a/backend/windmill-worker/src/worker_utils.rs b/backend/windmill-worker/src/worker_utils.rs index e1ea849999..4da318d37b 100644 --- a/backend/windmill-worker/src/worker_utils.rs +++ b/backend/windmill-worker/src/worker_utils.rs @@ -125,6 +125,7 @@ async fn update_worker_ping_full_inner( client .post::<_, ()>( UPDATE_PING_URL, + None, &Ping { last_job_executed: None, last_job_workspace_id: None, @@ -190,6 +191,7 @@ pub async fn insert_ping( client .post::<_, ()>( UPDATE_PING_URL, + None, &Ping { last_job_executed: None, last_job_workspace_id: None, @@ -249,6 +251,7 @@ pub async fn update_worker_ping_from_job( client .post::( UPDATE_PING_URL, + None, &Ping { last_job_executed: Some(job_id.clone()), last_job_workspace_id: Some(w_id.to_string()), @@ -287,6 +290,7 @@ pub async fn ping_job_status( client .post( &format!("/api/agent_workers/ping_job_status/{}", job_id), + None, &PingJobStatus { mem_peak, current_mem }, ) .await diff --git a/benchmarks/lib.ts b/benchmarks/lib.ts index cde9bc5320..45de71358f 100644 --- a/benchmarks/lib.ts +++ b/benchmarks/lib.ts @@ -2,7 +2,7 @@ import { sleep } from "https://deno.land/x/sleep@v1.2.1/mod.ts"; import * as windmill from "https://deno.land/x/windmill@v1.174.0/mod.ts"; import * as api from "https://deno.land/x/windmill@v1.174.0/windmill-api/index.ts"; -export const VERSION = "v1.494.0"; +export const VERSION = "v1.498.0"; export async function login(email: string, password: string): Promise { return await windmill.UserService.login({ diff --git a/cli/main.ts b/cli/main.ts index c5b67cc26a..3a505bbc18 100644 --- a/cli/main.ts +++ b/cli/main.ts @@ -63,7 +63,7 @@ export { // } // }); -export const VERSION = "1.494.0"; +export const VERSION = "1.498.0"; const command = new Command() .name("wmill") diff --git a/cli/metadata.ts b/cli/metadata.ts index ee39d0b9da..d2d37aa22d 100644 --- a/cli/metadata.ts +++ b/cli/metadata.ts @@ -40,7 +40,7 @@ export class LockfileGenerationError extends Error { export async function generateAllMetadata() {} function findClosestRawReqs( - lang: "bun" | "python3" | "php" | undefined, + lang: "bun" | "python3" | "php" | "go" | undefined, remotePath: string, globalDeps: GlobalDeps ): string | undefined { @@ -72,6 +72,15 @@ function findClosestRawReqs( bestCandidate = { k, v }; } }); + } else if (lang == "go") { + Object.entries(globalDeps.goMods).forEach(([k, v]) => { + if ( + remotePath.startsWith(k) && + k.length >= (bestCandidate?.k ?? "").length + ) { + bestCandidate = { k, v }; + } + }); } // @ts-ignore return bestCandidate?.v; @@ -193,14 +202,14 @@ export async function generateScriptMetadataInternal( const language = inferContentTypeFromFilePath(scriptPath, opts.defaultTs); const rawReqs = findClosestRawReqs( - language as "bun" | "python3" | "php" | undefined, + language as "bun" | "python3" | "php" | "go" | undefined, scriptPath, globalDeps ); if (rawReqs) { log.info( (await blueColor())( - `Found raw requirements (package.json/requirements.txt/composer.json) for ${scriptPath}, using it` + `Found raw requirements (package.json/requirements.txt/composer.json/go.mod) for ${scriptPath}, using it` ) ); } diff --git a/cli/script.ts b/cli/script.ts index e12c15d9ba..87e4f85354 100644 --- a/cli/script.ts +++ b/cli/script.ts @@ -869,11 +869,13 @@ export type GlobalDeps = { pkgs: Record; reqs: Record; composers: Record; + goMods: Record; }; export async function findGlobalDeps(): Promise { const pkgs: { [key: string]: string } = {}; const reqs: { [key: string]: string } = {}; const composers: { [key: string]: string } = {}; + const goMods: { [key: string]: string } = {}; const els = await FSFSElement(Deno.cwd(), [], false); for await (const entry of readDirRecursiveWithIgnore((p, isDir) => { p = SEP + p; @@ -882,21 +884,24 @@ export async function findGlobalDeps(): Promise { !( p.endsWith(SEP + "package.json") || p.endsWith(SEP + "requirements.txt") || - p.endsWith(SEP + "composer.json") + p.endsWith(SEP + "composer.json") || + p.endsWith(SEP + "go.mod") ) ); }, els)) { if (entry.isDirectory || entry.ignored) continue; const content = await entry.getContentText(); if (entry.path.endsWith("package.json")) { - pkgs[entry.path.substring(0, entry.path.length - 12)] = content; + pkgs[entry.path.substring(0, entry.path.length - "package.json".length)] = content; } else if (entry.path.endsWith("requirements.txt")) { - reqs[entry.path.substring(0, entry.path.length - 16)] = content; + reqs[entry.path.substring(0, entry.path.length - "requirements.txt".length)] = content; } else if (entry.path.endsWith("composer.json")) { - composers[entry.path.substring(0, entry.path.length - 13)] = content; + composers[entry.path.substring(0, entry.path.length - "composer.json".length)] = content; + } else if (entry.path.endsWith("go.mod")) { + goMods[entry.path.substring(0, entry.path.length - "go.mod".length)] = content; } } - return { pkgs, reqs, composers }; + return { pkgs, reqs, composers, goMods }; } async function generateMetadata( opts: GlobalOptions & { diff --git a/flake.nix b/flake.nix index 10a1e26c14..8056c201c5 100644 --- a/flake.nix +++ b/flake.nix @@ -22,6 +22,15 @@ "rustfmt" ]; }; + patchedClang = pkgs.llvmPackages_18.clang.overrideAttrs (oldAttrs: { + postFixup = '' + # Copy the original postFixup logic but skip add-hardening.sh + ${oldAttrs.postFixup or ""} + + # Remove the line that substitutes add-hardening.sh + sed -i 's/.*source.*add-hardening\.sh.*//' $out/bin/clang + ''; + }); buildInputs = with pkgs; [ openssl openssl.dev @@ -64,16 +73,24 @@ in { # Enter by `nix develop .#wasm` devShells."wasm" = pkgs.mkShell { + # Explicitly set paths for headers and linker + shellHook = '' + export CC=${patchedClang}/bin/clang + ''; buildInputs = buildInputs ++ (with pkgs; [ (rust-bin.nightly.latest.default.override { extensions = [ "rust-src" # for rust-analyzer "rust-analyzer" ]; - targets = [ "wasm32-unknown-unknown" ]; + targets = + [ "wasm32-unknown-unknown" "wasm32-unknown-emscripten" ]; }) wasm-pack deno + emscripten + # Needed for extra dependencies + glibc_multi ]); }; @@ -81,6 +98,7 @@ buildInputs = buildInputs ++ (with pkgs; [ # Essentials rust + cargo-watch cargo-sweep git xcaddy @@ -94,6 +112,9 @@ python3 python3Packages.pip uv + poetry + pyright + openapi-python-client # Other languages deno @@ -146,6 +167,8 @@ '') (pkgs.writeScriptBin "wm" '' cd ./frontend + npm install + npm run generate-backend-client npm run dev $* '') (pkgs.writeScriptBin "wm-minio" '' @@ -166,7 +189,6 @@ echo "bucket: wmill" echo "endpoint: http://localhost:9000" '') - ]; inherit PKG_CONFIG_PATH RUSTY_V8_ARCHIVE; @@ -179,6 +201,7 @@ RUSTC_WRAPPER = "${pkgs.sccache}/bin/sccache"; DENO_PATH = "${pkgs.deno}/bin/deno"; GO_PATH = "${pkgs.go}/bin/go"; + PHP_PATH = "${pkgs.php}/bin/php"; BUN_PATH = "${pkgs.bun}/bin/bun"; UV_PATH = "${pkgs.uv}/bin/uv"; NU_PATH = "${pkgs.nushell}/bin/nu"; @@ -194,7 +217,7 @@ ORACLE_LIB_DIR = "${pkgs.oracle-instantclient.lib}/lib"; ANSIBLE_PLAYBOOK_PATH = "${pkgs.ansible}/bin/ansible-playbook"; ANSIBLE_GALAXY_PATH = "${pkgs.ansible}/bin/ansible-galaxy"; - RUST_LOG = "debug"; + # RUST_LOG = "debug"; SQLX_OFFLINE = "true"; # See this issue: https://github.com/NixOS/nixpkgs/issues/370494 diff --git a/frontend/README_DEV.md b/frontend/README_DEV.md index ba90278be4..88c18440cb 100644 --- a/frontend/README_DEV.md +++ b/frontend/README_DEV.md @@ -1,21 +1,83 @@ -# Developing +# Windmill's build guide -[Using Nix](#nix), otherwise: +## Using Nix (Recommended) -In the `frontend/` directory: +Nix will manage all environment variables, packages and other configuration that you would usually do manually. -- install the dependencies with `npm install` (or `pnpm install` or `yarn`) -- generate the windmill client: +**Prerequisites** + +- Install [Nix](https://github.com/DeterminateSystems/nix-installer). +- Install Docker. +- Optionally install [direnv](https://direnv.net/docs/installation.html). + +That's it! You are ready to go. + +> Using **direnv** is highly recommended, since it can load shell automatically based on your CWD. It also can give you hints. + +### Development +```bash +# enter a dev shell containing all necessary packages. `direnv allow` if direnv is installed. +nix develop +## or ignore if you have `direnv` + +# Start db (if not started already) +sudo docker compose up db -d + +# run the frontend. +wm + +# In an other shell: +# +nix develop +## or ignore if you have `direnv` + +cd backend +# You don't need to install anything extra. All dependencies are already in place! +cargo run --features all_languages +``` + +The default proxy is setup to use the local backend: . + +### wm-* Commands + +Nix shell provides you with several helper commands prefixed with `wm-` ```bash +# Start minio server (implements S3) +wm-minio +# Note: You will need access to EE private repo in order to compile, don't forget "enterprise" and "parquet" freatures as well. + +# Generate keys for local dev. +wm-minio-keys +# Minio data as well as generated keys are stored in `backend/.minio-data` +``` + +You can read about all others commands individually in [flake.nix](../flake.nix). + +### dev.nu + +In some places we have `dev.nu` files. They can help you developing and testing specific features. Their functionality depends on context, but you can get more info by running it with `--help` flag. + +### Running on NixOS + +It is recommended to use [nix-alien](https://github.com/thiagokokada/nix-alien) for running compiled windmill binaries. We need this because sometimes windmill may fetch unpatched binaries that are not compatible with NixOS. + +## Traditional instructions + +### Frontend only + +```bash +cd frontend/ + +# Install dependencies. +npm install # or pnpm or yarn or bun + +# Generate windmill client. npm run generate-backend-client ## on mac use npm run generate-backend-client-mac -``` -Once the dependencies are installed, just start the dev server: - -```bash +# Start dev server npm run dev ``` @@ -27,9 +89,9 @@ You can configure another proxy to use like so: REMOTE=http://127.0.0.1:8000 REMOTE_LSP=http://127.0.0.1:3001 npm run dev ``` -## Use a Local backend +### Use a Local backend -### 1. Backend is run by docker +#### 1. Backend is run by docker ```bash docker build . -t windmill @@ -40,7 +102,23 @@ docker compose up db windmill_server windmill_worker REMOTE=http://localhost REMOTE_LSP=http://localhost npm run dev ``` -### 2. Backend is run by cargo +#### 2. Backend is run by docker, but built from the local source code + +Sometimes it is important to build docker image for your branch locally. It is crucial part of testing, since local environment may differ from the containerized one. + +That's why we provide [docker/dev.nu](../docker/dev.nu). It is helper that can build images locally and execute them. + +it can build the image and run on local repository. + +```bash +# Issue the build +docker/dev.nu up --features "python,static_frontend" docker/DockerfileNsjail --rebuild +# Will create and run `main__nsjail__python-static_frontend` +``` + +If you develop wasm parser for new language you can also pass `--wasm-pkg ` and it will include local parser to the image. For more information please see the script directly or run it with `--help` flag. + +#### 3. Backend is run by cargo **Prerequisites** @@ -87,13 +165,7 @@ In the frontend folder: REMOTE=http://127.0.0.1:8000 REMOTE_LSP=http://127.0.0.1:3001 npm run dev ``` -## Building - -```bash -NODE_OPTIONS=--max_old_space_size=8096 npm run build -``` - -## Formatting +### Formatting This project uses [prettier](https://prettier.io/docs/en/install.html) and [prettier-plugin-svelte](https://github.com/sveltejs/prettier-plugin-svelte), be @@ -116,15 +188,15 @@ Recommended config for VS Code: - turn _format on save_ on -## Building +### Building frontend The project is built with [SvelteKit](https://kit.svelte.dev/) and uses as output static files. There are others adapters for sveltekit, but we use the static adapter. To build the frontend as static assets, use: -``` -npm run build +```bash +NODE_OPTIONS=--max_old_space_size=8096 npm run build ``` The output is in the `build` folder. @@ -139,27 +211,7 @@ which will generate an index.html and allow you to serve the frontend with any s Env variables used for build are set in .env file. See [https://vitejs.dev/guide/env-and-mode.html#env-files](https://vitejs.dev/guide/env-and-mode.html#env-files) for more details. -## Nix - -Windmill use [nix flakes](https://nixos.wiki/wiki/Flakes): -```bash -nix run github:windmill-labs/windmill -``` - -### Development -```bash -nix develop # enter a dev shell containing all necessary packages. - -wm-setup # build the frontend and setup the database. -wm # run the frontend. - -# In an other shell: -nix develop -cd backend -cargo run -``` - -### Updating [`flake.nix`](../flake.nix) +## Updating [`flake.nix`](../flake.nix) ```bash nix flake update # update the lock file. diff --git a/frontend/npm b/frontend/npm deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 363691c469..02bb998500 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -1,12 +1,12 @@ { "name": "windmill-components", - "version": "1.494.0", + "version": "1.498.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "windmill-components", - "version": "1.494.0", + "version": "1.498.0", "hasInstallScript": true, "license": "AGPL-3.0", "dependencies": { @@ -22,9 +22,12 @@ "@leeoniya/ufuzzy": "^1.0.8", "@popperjs/core": "^2.11.6", "@redocly/json-to-json-schema": "^0.0.1", + "@scalar/openapi-parser": "^0.15.0", "@tanstack/svelte-table": "npm:tanstack-table-8-svelte-5@^0.1", + "@tutorlatin/svelte-tiny-virtual-list": "^3.0.2", "@windmill-labs/svelte-dnd-action": "^0.9.48", - "@xyflow/svelte": "^0.1.15", + "@xterm/addon-fit": "^0.10.0", + "@xyflow/svelte": "^1.0.0", "ag-charts-community": "^9.0.1", "ag-charts-enterprise": "^9.0.1", "ag-grid-community": "^31.3.4", @@ -52,6 +55,7 @@ "monaco-vim": "^0.4.1", "ol": "^7.4.0", "openai": "^4.87.1", + "openapi-types": "^12.1.3", "p-limit": "^6.1.0", "panzoom": "^9.4.3", "pdfjs-dist": "4.8.69", @@ -79,11 +83,14 @@ "windmill-parser-wasm-ts": "^1.486.1", "windmill-parser-wasm-yaml": "^1.429.0", "windmill-sql-datatype-parser-wasm": "^1.318.0", + "xterm": "^5.3.0", + "xterm-readline": "^1.1.2", "y-monaco": "^0.1.4", "y-websocket": "^1.5.4", - "yaml": "^2.3.4", + "yaml": "^2.8.0", "yjs": "^13.6.7", - "zod": "^3.24.2" + "zod": "^3.24.2", + "zod-to-json-schema": "^3.24.5" }, "devDependencies": { "@floating-ui/core": "^1.3.1", @@ -118,7 +125,6 @@ "postcss-load-config": "^4.0.1", "prettier": "^3.1.0", "prettier-plugin-svelte": "^3.3.3", - "simple-svelte-autocomplete": "^2.5.1", "style-to-object": "^0.4.1", "stylelint-config-recommended": "^13.0.0", "svelte": "^5.0.0", @@ -3000,6 +3006,59 @@ "win32" ] }, + "node_modules/@scalar/openapi-parser": { + "version": "0.15.0", + "resolved": "https://registry.npmjs.org/@scalar/openapi-parser/-/openapi-parser-0.15.0.tgz", + "integrity": "sha512-49u1sE+P+72nbt/9p7flBS/g7wbapTGAplcQ1FZxa4La56OMuShCl4qSSdw8lPNZjUZNTLcw7ExVOGI9p1kkOA==", + "license": "MIT", + "dependencies": { + "ajv": "^8.17.1", + "ajv-draft-04": "^1.0.0", + "ajv-formats": "^3.0.1", + "jsonpointer": "^5.0.1", + "leven": "^4.0.0", + "yaml": "^2.4.5" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/@scalar/openapi-parser/node_modules/ajv": { + "version": "8.17.1", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.17.1.tgz", + "integrity": "sha512-B/gBuNg5SiMTrPkC+A2+cW0RszwxYmn6VYxB/inlBStS5nx6xHIt/ehKRhIMhqusl7a8LjQoZnjCs5vhwxOQ1g==", + "license": "MIT", + "dependencies": { + "fast-deep-equal": "^3.1.3", + "fast-uri": "^3.0.1", + "json-schema-traverse": "^1.0.0", + "require-from-string": "^2.0.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, + "node_modules/@scalar/openapi-parser/node_modules/ajv-draft-04": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/ajv-draft-04/-/ajv-draft-04-1.0.0.tgz", + "integrity": "sha512-mv00Te6nmYbRp5DCwclxtt7yV/joXJPGS7nM+97GdxvuttCOfgI3K4U25zboyeX0O+myI8ERluxQe5wljMmVIw==", + "license": "MIT", + "peerDependencies": { + "ajv": "^8.5.0" + }, + "peerDependenciesMeta": { + "ajv": { + "optional": true + } + } + }, + "node_modules/@scalar/openapi-parser/node_modules/json-schema-traverse": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", + "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==", + "license": "MIT" + }, "node_modules/@smithy/types": { "version": "4.2.0", "resolved": "https://registry.npmjs.org/@smithy/types/-/types-4.2.0.tgz", @@ -3017,11 +3076,11 @@ "integrity": "sha512-VqAAkydywPpkw63WQhPVKCD3SdwXuihCUVZbbiY3SfSTGQyHmwRoq27y4dmJdZuJwd5JIlQoMPyGvMbUPY0RKQ==" }, "node_modules/@svelte-put/shortcut": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/@svelte-put/shortcut/-/shortcut-3.1.1.tgz", - "integrity": "sha512-2L5EYTZXiaKvbEelVkg5znxqvfZGZai3m97+cAiUBhLZwXnGtviTDpHxOoZBsqz41szlfRMcamW/8o0+fbW3ZQ==", + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@svelte-put/shortcut/-/shortcut-4.1.0.tgz", + "integrity": "sha512-wImNEIkbxAIWFqlfuhcbC+jRPDeRa/uJGIXHMEVVD+jqL9xCwWNnkGQJ6Qb2XVszuRLHlb8SGZDL3Io/h3vs8w==", "peerDependencies": { - "svelte": "^3.55.0 || ^4.0.0 || ^5.0.0" + "svelte": "^5.1.0" } }, "node_modules/@sveltejs/acorn-typescript": { @@ -3215,6 +3274,17 @@ "node": ">=10.13.0" } }, + "node_modules/@tutorlatin/svelte-tiny-virtual-list": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/@tutorlatin/svelte-tiny-virtual-list/-/svelte-tiny-virtual-list-3.0.2.tgz", + "integrity": "sha512-Zts/GUrU6aPgmCrL6sGiD1EoiQxUU+r7Q2g9pckEUIDybNkluV2pEYWUsQLQ+28IJlSSHTopv0fhKs3qMUxqEw==", + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "svelte": "^5.0.0" + } + }, "node_modules/@types/cookie": { "version": "0.6.0", "resolved": "https://registry.npmjs.org/@types/cookie/-/cookie-0.6.0.tgz", @@ -3795,23 +3865,38 @@ "svelte": ">=3.23.0 || ^5.0.0-next.0" } }, + "node_modules/@xterm/addon-fit": { + "version": "0.10.0", + "resolved": "https://registry.npmjs.org/@xterm/addon-fit/-/addon-fit-0.10.0.tgz", + "integrity": "sha512-UFYkDm4HUahf2lnEyHvio51TNGiLK66mqP2JoATy7hRZeXaGMRDr00JiSF7m63vR5WKATF605yEggJKsw0JpMQ==", + "license": "MIT", + "peerDependencies": { + "@xterm/xterm": "^5.0.0" + } + }, + "node_modules/@xterm/xterm": { + "version": "5.5.0", + "resolved": "https://registry.npmjs.org/@xterm/xterm/-/xterm-5.5.0.tgz", + "integrity": "sha512-hqJHYaQb5OptNunnyAnkHyM8aCjZ1MEIDTQu1iIbbTD/xops91NB5yq1ZK/dC2JDbVWtF23zUtl9JE2NqwT87A==", + "license": "MIT", + "peer": true + }, "node_modules/@xyflow/svelte": { - "version": "0.1.39", - "resolved": "https://registry.npmjs.org/@xyflow/svelte/-/svelte-0.1.39.tgz", - "integrity": "sha512-QZ5mzNysvJeJW7DxmqI4Urhhef9tclqtPr7WAS5zQF5Gk6k9INwzey4CYNtEZo8XMj9H8lzgoJRmgMPnJEc1kw==", + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@xyflow/svelte/-/svelte-1.0.2.tgz", + "integrity": "sha512-biNnzWRmKF/m7t7iY9wt3/KhoTyA8XGbS0Vix3HCL9vrULDF14R1VsMIOxKZib1GJfG+f1/cD2dQ9kplfPLs/Q==", "dependencies": { - "@svelte-put/shortcut": "3.1.1", - "@xyflow/system": "0.0.59", - "classcat": "^5.0.4" + "@svelte-put/shortcut": "^4.1.0", + "@xyflow/system": "0.0.61" }, "peerDependencies": { - "svelte": "^3.0.0 || ^4.0.0 || ^5.0.0" + "svelte": "^5.25.0" } }, "node_modules/@xyflow/system": { - "version": "0.0.59", - "resolved": "https://registry.npmjs.org/@xyflow/system/-/system-0.0.59.tgz", - "integrity": "sha512-+xgqYhoBv5F10TQx0SiKZR/DcWtuxFYR+e/LluHb7DMtX4SsMDutZWEJ4da4fDco25jZxw5G9fOlmk7MWvYd5Q==", + "version": "0.0.61", + "resolved": "https://registry.npmjs.org/@xyflow/system/-/system-0.0.61.tgz", + "integrity": "sha512-TsZG/Ez8dzxX6/Ol44LvFqVZsYvyz6dpDlAQZZk6hTL7JLGO5vN3dboRJqMwU8/Qtr5IEv5YBzojjAwIqW1HCA==", "dependencies": { "@types/d3-drag": "^3.0.7", "@types/d3-selection": "^3.0.10", @@ -3933,6 +4018,45 @@ "url": "https://github.com/sponsors/epoberezkin" } }, + "node_modules/ajv-formats": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/ajv-formats/-/ajv-formats-3.0.1.tgz", + "integrity": "sha512-8iUql50EUR+uUcdRQ3HDqa6EVyo3docL8g5WJ3FNcWmu62IbkGUue/pEyLBW8VGKKucTPgqeks4fIU1DA4yowQ==", + "license": "MIT", + "dependencies": { + "ajv": "^8.0.0" + }, + "peerDependencies": { + "ajv": "^8.0.0" + }, + "peerDependenciesMeta": { + "ajv": { + "optional": true + } + } + }, + "node_modules/ajv-formats/node_modules/ajv": { + "version": "8.17.1", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.17.1.tgz", + "integrity": "sha512-B/gBuNg5SiMTrPkC+A2+cW0RszwxYmn6VYxB/inlBStS5nx6xHIt/ehKRhIMhqusl7a8LjQoZnjCs5vhwxOQ1g==", + "license": "MIT", + "dependencies": { + "fast-deep-equal": "^3.1.3", + "fast-uri": "^3.0.1", + "json-schema-traverse": "^1.0.0", + "require-from-string": "^2.0.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, + "node_modules/ajv-formats/node_modules/json-schema-traverse": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", + "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==", + "license": "MIT" + }, "node_modules/amator": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/amator/-/amator-1.1.0.tgz", @@ -3953,7 +4077,6 @@ "version": "5.0.1", "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", - "dev": true, "engines": { "node": ">=8" } @@ -4597,11 +4720,6 @@ "consola": "^3.2.3" } }, - "node_modules/classcat": { - "version": "5.0.5", - "resolved": "https://registry.npmjs.org/classcat/-/classcat-5.0.5.tgz", - "integrity": "sha512-JhZUT7JFcQy/EzW605k/ktHtncoo9vnyW/2GspNYwFlN1C/WmjuV/xtS04e9SOkL2sTdw0VAZ2UGCcQ9lR6p6w==" - }, "node_modules/clone": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/clone/-/clone-2.1.2.tgz", @@ -5485,8 +5603,7 @@ "node_modules/emoji-regex": { "version": "8.0.0", "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", - "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", - "dev": true + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==" }, "node_modules/encoding-down": { "version": "6.3.0", @@ -6043,8 +6160,7 @@ "node_modules/fast-deep-equal": { "version": "3.1.3", "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", - "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==", - "dev": true + "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==" }, "node_modules/fast-diff": { "version": "1.1.2", @@ -6103,7 +6219,6 @@ "version": "3.0.6", "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.0.6.tgz", "integrity": "sha512-Atfo14OibSv5wAp4VWNsFYE1AchQRTv9cBGWET4pZWHzYshFSS9NQI6I57rdKn9croWVMbYFbLhJ+yJvmZIIHw==", - "dev": true, "funding": [ { "type": "github", @@ -6113,8 +6228,7 @@ "type": "opencollective", "url": "https://opencollective.com/fastify" } - ], - "peer": true + ] }, "node_modules/fastest-levenshtein": { "version": "1.0.16", @@ -7179,7 +7293,6 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", - "dev": true, "engines": { "node": ">=8" } @@ -7361,6 +7474,15 @@ "resolved": "https://registry.npmjs.org/json-stringify-pretty-compact/-/json-stringify-pretty-compact-2.0.0.tgz", "integrity": "sha512-WRitRfs6BGq4q8gTgOy4ek7iPFXjbra0H3PmDLKm2xnZ+Gh1HUhiKGgCZkSPNULlP7mvfu6FV/mOLhCarspADQ==" }, + "node_modules/jsonpointer": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/jsonpointer/-/jsonpointer-5.0.1.tgz", + "integrity": "sha512-p/nXbhSEcu3pZRdkW1OfJhpsVtW1gd4Wa1fnQc9YLiTfAjn0312eMKimbdIQzuZl9aa9xUGaRlP9T/CJE/ditQ==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/keyv": { "version": "4.5.4", "resolved": "https://registry.npmjs.org/keyv/-/keyv-4.5.4.tgz", @@ -7540,6 +7662,18 @@ "node": ">=6" } }, + "node_modules/leven": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/leven/-/leven-4.0.0.tgz", + "integrity": "sha512-puehA3YKku3osqPlNuzGDUHq8WpwXupUg1V6NXdV38G+gr+gkBwFC8g1b/+YcIvp8gnqVIus+eJCH/eGsRmJNw==", + "license": "MIT", + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/levn": { "version": "0.4.1", "resolved": "https://registry.npmjs.org/levn/-/levn-0.4.1.tgz", @@ -9211,6 +9345,12 @@ "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz", "integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==" }, + "node_modules/openapi-types": { + "version": "12.1.3", + "resolved": "https://registry.npmjs.org/openapi-types/-/openapi-types-12.1.3.tgz", + "integrity": "sha512-N4YtSYJqghVu4iek2ZUvcN/0aqH1kRDuNqzcycDxhOUpg7GdvLa2F3DgS6yBNhInhv2r/6I0Flkn7CqL8+nIcw==", + "license": "MIT" + }, "node_modules/optionator": { "version": "0.9.4", "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.9.4.tgz", @@ -10714,8 +10854,6 @@ "version": "2.0.2", "resolved": "https://registry.npmjs.org/require-from-string/-/require-from-string-2.0.2.tgz", "integrity": "sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==", - "dev": true, - "peer": true, "engines": { "node": ">=0.10.0" } @@ -11012,12 +11150,6 @@ "simple-concat": "^1.0.0" } }, - "node_modules/simple-svelte-autocomplete": { - "version": "2.5.2", - "resolved": "https://registry.npmjs.org/simple-svelte-autocomplete/-/simple-svelte-autocomplete-2.5.2.tgz", - "integrity": "sha512-6+kZP3XQGb7zs0djTWoFv8tQorBZdv64YfCKrlsYIYBCFsEeikSiNsBSSYw1Om+LYmFeMEOAYSSKoEw+qpnBXQ==", - "dev": true - }, "node_modules/sirv": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/sirv/-/sirv-3.0.1.tgz", @@ -11163,7 +11295,6 @@ "version": "4.2.3", "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", - "dev": true, "dependencies": { "emoji-regex": "^8.0.0", "is-fullwidth-code-point": "^3.0.0", @@ -11192,7 +11323,6 @@ "version": "6.0.1", "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", - "dev": true, "dependencies": { "ansi-regex": "^5.0.1" }, @@ -13029,6 +13159,25 @@ "node": ">=0.4" } }, + "node_modules/xterm": { + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/xterm/-/xterm-5.3.0.tgz", + "integrity": "sha512-8QqjlekLUFTrU6x7xck1MsPzPA571K5zNqWm0M0oroYEWVOptZ0+ubQSkQ3uxIEhcIHRujJy6emDWX4A7qyFzg==", + "deprecated": "This package is now deprecated. Move to @xterm/xterm instead.", + "license": "MIT" + }, + "node_modules/xterm-readline": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/xterm-readline/-/xterm-readline-1.1.2.tgz", + "integrity": "sha512-1+W2nVuQvCYz9OUYwFBiolrSQUui51aDDyacKXt4PuxeBHqzvabQEJ2kwdBDzsmOjz5BwlDTAjJmYpH2OGqLFA==", + "license": "MIT", + "dependencies": { + "string-width": "^4" + }, + "peerDependencies": { + "@xterm/xterm": "^5.5.0" + } + }, "node_modules/y-leveldb": { "version": "0.1.2", "resolved": "https://registry.npmjs.org/y-leveldb/-/y-leveldb-0.1.2.tgz", @@ -13129,6 +13278,7 @@ "version": "2.8.0", "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.8.0.tgz", "integrity": "sha512-4lLa/EcQCB0cJkyts+FpIRx5G/llPxfP6VQU5KByHEhLxY3IJCH0f0Hy1MHI8sClTvsIb8qwRJ6R/ZdlDJ/leQ==", + "license": "ISC", "bin": { "yaml": "bin.mjs" }, @@ -13192,6 +13342,15 @@ "url": "https://github.com/sponsors/colinhacks" } }, + "node_modules/zod-to-json-schema": { + "version": "3.24.5", + "resolved": "https://registry.npmjs.org/zod-to-json-schema/-/zod-to-json-schema-3.24.5.tgz", + "integrity": "sha512-/AuWwMP+YqiPbsJx5D6TfgRTc4kTLjsh5SOcd4bLsfUg2RcEXrFMJl1DGgdHy2aCfsIA/cr/1JM0xcB2GZji8g==", + "license": "ISC", + "peerDependencies": { + "zod": "^3.24.1" + } + }, "node_modules/zstddec": { "version": "0.1.0", "resolved": "https://registry.npmjs.org/zstddec/-/zstddec-0.1.0.tgz", diff --git a/frontend/package.json b/frontend/package.json index 4666520da6..bee87418b0 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "windmill-components", - "version": "1.494.0", + "version": "1.498.0", "scripts": { "dev": "vite dev", "build": "vite build", @@ -50,7 +50,6 @@ "postcss-load-config": "^4.0.1", "prettier": "^3.1.0", "prettier-plugin-svelte": "^3.3.3", - "simple-svelte-autocomplete": "^2.5.1", "style-to-object": "^0.4.1", "stylelint-config-recommended": "^13.0.0", "svelte": "^5.0.0", @@ -81,19 +80,21 @@ "dependencies": { "@aws-crypto/sha256-js": "^4.0.0", "@codingame/monaco-vscode-configuration-service-override": "~16.1.1", + "@codingame/monaco-vscode-editor-api": "~16.1.1", "@codingame/monaco-vscode-standalone-css-language-features": "~16.1.1", "@codingame/monaco-vscode-standalone-html-language-features": "^16.1.1", "@codingame/monaco-vscode-standalone-json-language-features": "~16.1.1", "@codingame/monaco-vscode-standalone-languages": "~16.1.1", "@codingame/monaco-vscode-standalone-typescript-language-features": "~16.1.1", - "@codingame/monaco-vscode-editor-api": "~16.1.1", "@json2csv/plainjs": "^7.0.6", "@leeoniya/ufuzzy": "^1.0.8", "@popperjs/core": "^2.11.6", "@redocly/json-to-json-schema": "^0.0.1", + "@scalar/openapi-parser": "^0.15.0", "@tanstack/svelte-table": "npm:tanstack-table-8-svelte-5@^0.1", "@windmill-labs/svelte-dnd-action": "^0.9.48", - "@xyflow/svelte": "^0.1.15", + "@xyflow/svelte": "^1.0.0", + "@xterm/addon-fit": "^0.10.0", "ag-charts-community": "^9.0.1", "ag-charts-enterprise": "^9.0.1", "ag-grid-community": "^31.3.4", @@ -121,6 +122,7 @@ "monaco-vim": "^0.4.1", "ol": "^7.4.0", "openai": "^4.87.1", + "openapi-types": "^12.1.3", "p-limit": "^6.1.0", "panzoom": "^9.4.3", "pdfjs-dist": "4.8.69", @@ -132,6 +134,7 @@ "svelte-exmarkdown": "^5.0.0", "svelte-infinite-loading": "^1.4.0", "svelte-tiny-virtual-list": "^2.0.5", + "@tutorlatin/svelte-tiny-virtual-list": "^3.0.2", "tailwind-merge": "^1.13.2", "vscode": "npm:@codingame/monaco-vscode-extension-api@~16.1.1", "vscode-languageclient": "~9.0.1", @@ -148,11 +151,14 @@ "windmill-parser-wasm-ts": "^1.486.1", "windmill-parser-wasm-yaml": "^1.429.0", "windmill-sql-datatype-parser-wasm": "^1.318.0", + "xterm": "^5.3.0", + "xterm-readline": "^1.1.2", "y-monaco": "^0.1.4", "y-websocket": "^1.5.4", - "yaml": "^2.3.4", + "yaml": "^2.8.0", "yjs": "^13.6.7", - "zod": "^3.24.2" + "zod": "^3.24.2", + "zod-to-json-schema": "^3.24.5" }, "peerDependencies": { "svelte": "^4.0.0" diff --git a/frontend/src/lib/autosize.ts b/frontend/src/lib/autosize.ts index 9ea2f925cd..b47a2e57b7 100644 --- a/frontend/src/lib/autosize.ts +++ b/frontend/src/lib/autosize.ts @@ -1,85 +1,90 @@ -export const action = (node) => { - /* Constants */ - const update = new Event('update') +import { tick } from 'svelte' - /* Functions */ - const init = () => { - addStyles() - observeElement() - addEventListeners() - setInitialHeight() - } +type TextArea = HTMLTextAreaElement - const dispatchUpdateEvent = () => { - node.dispatchEvent(update) - } +export const autosize = (node: TextArea) => { + /* ------------------------------------------------------------------ + * Constants + * ---------------------------------------------------------------- */ + const UPDATE_EVENT = new Event('update') + const MIN_HEIGHT = 30 // px + const EXTRA = 2 // px added to scrollHeight - const setInitialHeight = () => { - let height = 0 - if (node.value) { - height = Math.max(node.scrollHeight ?? 0, 30) + 2 - } else { - if (node.placeholder) { - node.value = node.placeholder - height = Math.max(node.scrollHeight ?? 0, 30) + 2 - node.value = '' - } else { - node.value = '|' - node.style.height = '0px' - height = Math.max(node.scrollHeight ?? 0, 30) + 2 - node.value = '' - } - } + let width = 0 - node.style.height = height + 'px' - } - - const setHeight = () => { + /* ------------------------------------------------------------------ + * Core resize routine + * ---------------------------------------------------------------- */ + const resize = () => { node.style.height = 'auto' - node.style.height = Math.max(node.scrollHeight ?? 0, 30) + 2 + 'px' + node.style.height = `${Math.max(node.scrollHeight, MIN_HEIGHT) + EXTRA}px` } - const addStyles = () => { - node.style.boxSizing = 'border-box' - } + /* ------------------------------------------------------------------ + * Patch `value` so programmatic changes trigger resize + * ---------------------------------------------------------------- */ + const proto = Object.getPrototypeOf(node) + const desc = Object.getOwnPropertyDescriptor(proto, 'value') - const observeElement = () => { - let elementPrototype = Object.getPrototypeOf(node) - let descriptor = Object.getOwnPropertyDescriptor(elementPrototype, 'value') + if (desc) { Object.defineProperty(node, 'value', { - get: function () { - return descriptor?.get?.apply(this, arguments as any) + get() { + return desc.get?.call(this) }, - set: function () { - descriptor?.set?.apply(this, arguments as any) - dispatchUpdateEvent() + set(v: unknown) { + desc.set?.call(this, v) + node.dispatchEvent(UPDATE_EVENT) } }) } - const addEventListeners = () => { - node.addEventListener('input', (e) => { - dispatchUpdateEvent() - }) - node.addEventListener('update', setHeight) - } + /* ------------------------------------------------------------------ + * Event listeners + * ---------------------------------------------------------------- */ + const onInput = () => node.dispatchEvent(UPDATE_EVENT) - const removeEventListeners = () => { - node.removeEventListener('input', dispatchUpdateEvent) - node.removeEventListener('update', setHeight) - } + node.addEventListener('input', onInput) + node.addEventListener('update', resize) - if (node.tagName.toLowerCase() !== 'textarea') { - throw new Error('svelte-textarea-auto-height can only be used on textarea elements.') - } else { - init() + /* ------------------------------------------------------------------ + * Inline styling + * ---------------------------------------------------------------- */ + node.style.boxSizing = 'border-box' - return { - destroy() { - removeEventListeners() - } + /* ------------------------------------------------------------------ + * Wait for DOM mount, then do an initial measure. + * If the + {:else if description == undefined || description == ''}
No description provided
@@ -587,7 +614,7 @@

Scopes

import type { EnumType } from '$lib/common' - import AutoComplete from 'simple-svelte-autocomplete' import { createEventDispatcher } from 'svelte' - import { twMerge } from 'tailwind-merge' + import Select from './Select.svelte' - export let disabled: boolean - export let value: any - export let enum_: EnumType - export let autofocus: boolean | null - export let defaultValue: string | undefined - export let valid: boolean - export let create: boolean - export let required: boolean - export let enumLabels: Record | undefined = undefined + interface Props { + disabled: boolean + value: string | undefined + enum_: EnumType + autofocus: boolean | null + defaultValue: string | undefined + valid: boolean + create: boolean + enumLabels?: Record | undefined + } + + let { + disabled, + value = $bindable(), + enum_, + autofocus, + defaultValue, + valid, + create, + enumLabels = undefined + }: Props = $props() const dispatch = createEventDispatcher() - let customItems: string[] = [] + let customItems: string[] = $state([]) - function onCreate(newItem: string) { - console.log(newItem) - customItems = [...customItems, newItem] + let items = $derived.by(() => { + const l = [...(enum_ ? enum_ : []), ...customItems].map((item) => ({ + value: item, + label: enumLabels?.[item] ?? item + })) + if (create && filterText && l.every((i) => i.value !== filterText)) { + l.push({ value: filterText, label: `Add new: ${filterText}` }) + } + return l + }) - return newItem - } + let filterText = $state('')
- (enumLabels ? enumLabels[val] ?? val : val)} - bind:selectedItem={value} - inputClassName={twMerge( - 'bg-surface-secondary flex', - valid - ? '' - : 'border border-red-700 border-opacity-30 focus:border-red-700 focus:border-opacity-30 bg-red-100' - )} - value={value ?? defaultValue} - hideArrow={true} - dropdownClassName="!text-sm !py-2 !rounded-sm !border-gray-200 !border !shadow-md !bg-surface-primary" - className="w-full" - noInputStyles - onFocus={() => { - dispatch('focus') - }} - onBlur={() => { - dispatch('blur') - }} - {create} - {required} - {onCreate} + { + onkeydown={() => { ignoreValueUndefined = true }} class={valid @@ -545,6 +636,9 @@ outerDivClass={'dark:!border-gray-500 !border-gray-300'} {disabled} bind:selected={value} + onremove={(e) => { + if (Array.isArray(value)) value = value.filter((v) => v !== e.option) + }} options={itemsType?.multiselect ?? []} selectedOptionsDraggable={true} onopen={() => { @@ -552,14 +646,17 @@ }} />
- {:else if itemsType?.enum != undefined && Array.isArray(itemsType?.enum) && Array.isArray(value)} + {:else if itemsType?.enum != undefined && Array.isArray(itemsType?.enum) && (Array.isArray(value) || value == undefined)}
{ + if (Array.isArray(value)) value = value.filter((v) => v !== e.option) + }} bind:selected={ - () => [...value], + () => [...(value ?? [])], (v) => { if (!deepEqual(v, value)) { value = v @@ -610,7 +707,7 @@ fileChanged(x, (val) => (value[i] = val))} + onchange={(x) => fileChanged(x, (val) => (value[i] = val))} multiple={false} /> {:else if itemsType?.type == 'object' && itemsType?.resourceType === undefined && itemsType?.properties === undefined} @@ -624,7 +721,6 @@ {/await} {:else if Array.isArray(itemsType?.enum)} { dispatch('focus') @@ -633,7 +729,7 @@ dispatch('blur') }} {defaultValue} - {valid} + valid={valid ?? true} {disabled} {autofocus} bind:value={value[i]} @@ -680,7 +776,7 @@ transition:fade|local={{ duration: 100 }} class="rounded-full p-1 bg-surface-secondary duration-200 hover:bg-surface-hover ml-2" aria-label="Clear" - on:click={() => { + onclick={() => { value = value.filter((_, index) => index !== i) redraw += 1 }} @@ -691,7 +787,7 @@ {/if} {/each} {#if value.length > itemsLimit} - {/if} @@ -754,16 +850,16 @@ args={otherArgs} {helperScript} bind:value - entrypoint={format.substring('dynselect_'.length)} + entrypoint={format?.substring('dynselect_'.length) ?? ''} /> {:else if inputCat == 'resource-object' && resourceTypes == undefined} Loading resource types... - {:else if inputCat == 'resource-object' && (resourceTypes == undefined || (format.split('-').length > 1 && resourceTypes.includes(format.substring('resource-'.length))))} + {:else if inputCat == 'resource-object' && (resourceTypes == undefined || (format && format?.split('-').length > 1 && resourceTypes.includes(format?.substring('resource-'.length))))} { @@ -771,7 +867,7 @@ }} {showSchemaExplorer} /> - {:else if inputCat == 'resource-object' && format.split('-').length > 1 && format + {:else if inputCat == 'resource-object' && format && format.split('-').length > 1 && format .replace('resource-', '') .replace('_', '') .toLowerCase() == 's3object'} @@ -855,11 +951,12 @@ : 'label' value = { ...toKeep, [tagKey]: detail } }} - let:item > - {#each oneOf as obj} - - {/each} + {#snippet children({ item })} + {#each oneOf as obj} + + {/each} + {/snippet} {#if oneOfSelected} {@const objIdx = oneOf.findIndex((o) => o.title === oneOfSelected)} @@ -873,13 +970,18 @@ {onlyMaskPassword} {disablePortal} {disabled} - schema={{ - properties: obj.properties, - order: obj.order, - $schema: '', - required: obj.required ?? [], - type: 'object' - }} + bind:schema={ + () => ({ + properties: obj.properties ?? {}, + order: obj.order, + $schema: '', + required: obj.required ?? [], + type: 'object' + }), + () => { + dispatch('nestedChange') + } + } bind:args={value} dndType={`nested-${title}`} hiddenArgs={['label', 'kind']} @@ -889,9 +991,6 @@ oneOf[objIdx].order = keys } }} - on:change={() => { - dispatch('nestedChange') - }} on:nestedChange {shouldDispatchChanges} /> @@ -909,7 +1008,12 @@ required: obj.required ?? [], type: 'object' }} - bind:args={value} + bind:args={ + () => value, + (v) => { + value = { ...v, [tagKey]: oneOfSelected } + } + } {shouldDispatchChanges} on:change={() => { dispatch('nestedChange') @@ -971,22 +1075,24 @@ {onlyMaskPassword} {disablePortal} {disabled} - schema={{ - properties, - $schema: '', - required: nestedRequired ?? [], - type: 'object', - order - }} + bind:schema={ + () => ({ + properties, + $schema: '', + required: nestedRequired ?? [], + type: 'object', + order + }), + (newSchema) => { + dispatch('nestedChange') + } + } bind:args={value} dndType={`nested-${title}`} on:reorder={(e) => { const keys = e.detail order = keys }} - on:change={() => { - dispatch('nestedChange') - }} diff={diffStatus && typeof diffStatus.diff === 'object' ? diffStatus.diff : {}} on:acceptChange={(e) => { dispatch('acceptChange', e.detail) @@ -1065,10 +1171,9 @@ {:else if inputCat == 'enum'}
fileChanged(x, (val) => (value = val))} + onchange={(x) => fileChanged(x, (val) => (value = val))} multiple={false} /> {#if value?.length} @@ -1144,8 +1249,8 @@ {:else if inputCat == 'email'} { + onfocus={(e) => { dispatch('focus') }} - on:blur={(e) => { + onblur={(e) => { dispatch('blur') }} use:autosize - on:keydown={onKeyDown} + onkeydown={onKeyDown} {disabled} class={twMerge( 'w-full', @@ -1197,10 +1302,10 @@ > {/key} {#if !disabled && itemPicker && extra?.['disableVariablePicker'] != true} - + {/if} @@ -1224,7 +1329,7 @@ {/if}
{/if} - + {@render actions?.()}
{#if !compact || (error && error != '')} diff --git a/frontend/src/lib/components/ArrayTypeNarrowing.svelte b/frontend/src/lib/components/ArrayTypeNarrowing.svelte index 3694b021b7..865aadc8ca 100644 --- a/frontend/src/lib/components/ArrayTypeNarrowing.svelte +++ b/frontend/src/lib/components/ArrayTypeNarrowing.svelte @@ -9,17 +9,25 @@ import EditableSchemaDrawer from './schema/EditableSchemaDrawer.svelte' import type { SchemaProperty } from '$lib/common' - export let canEditResourceType: boolean = false - export let originalType: string | undefined = undefined - export let itemsType: - | { - type?: 'string' | 'number' | 'bytes' | 'object' | 'resource' - contentEncoding?: 'base64' - enum?: string[] - resourceType?: string - properties?: { [name: string]: SchemaProperty } - } - | undefined + interface Props { + canEditResourceType?: boolean + originalType?: string | undefined + itemsType: + | { + type?: 'string' | 'number' | 'bytes' | 'object' | 'resource' + contentEncoding?: 'base64' + enum?: string[] + resourceType?: string + properties?: { [name: string]: SchemaProperty } + } + | undefined + } + + let { + canEditResourceType = false, + originalType = undefined, + itemsType = $bindable() + }: Props = $props() let selected: | 'string' @@ -29,7 +37,7 @@ | 'enum' | 'resource' | 's3object' - | undefined = + | undefined = $state( itemsType?.type != 'string' ? itemsType?.type == 'object' && itemsType?.resourceType == 's3object' ? 's3object' @@ -39,12 +47,13 @@ : itemsType?.contentEncoding == 'base64' ? 'bytes' : 'string' + ) - let schema = { + let schema = $state({ properties: itemsType?.properties || {}, order: Object.keys(itemsType?.properties || {}), required: Object.values(itemsType?.properties || {}).map((p) => p.required) - } + }) function updateItemsType() { itemsType = { @@ -59,7 +68,7 @@ + + + {#if !windmillBuiltins.includes(k) && k != 'slack'} + + {/if} +
+ + {/if} + {/if} + {/each} + +
+ + +
+
+ +
+ {:else if tab === 'oauth'} +
+ When one of the below options is set, you will be able to create a specific resource + containing a token automatically generated by the third-party provider. +
+ To test it after setting an oauth client, go to the Resources menu and create a new one of + the type of your oauth client (i.e. a 'github' resource if you set Github OAuth). +
Learn more
+
+
+ +
+ +
-
-
- When at least one of the below options is set, users will be able to login to Windmill via - their third-party account. -
To test SSO, the recommended workflow is to to save the settings and try to login in - an incognito window. - Learn more
-
-
- - - - - - - - - - - - {#each Object.keys(oauths) as k} - {#if !['authelia', 'authentik', 'google', 'microsoft', 'github', 'gitlab', 'jumpcloud', 'okta', 'auth0', 'keycloak', 'slack', 'kanidm', 'zitadel'].includes(k) && 'login_config' in oauths[k]} - {#if oauths[k]} + {#if oauths[k] && !('login_config' in oauths[k])} + {#if !['slack', 'teams'].includes(k) && oauths[k]}
- + { - delete oauths[k] - oauths = { ...oauths } + if (oauths) { + delete oauths[k] + oauths = { ...oauths } + } }} />
- - {#if !windmillBuiltins.includes(k) && k != 'slack'} - + {#if k === 'azure_oauth'} + + {:else if !windmillBuiltins.includes(k) && k != 'slack'} + + {/if} + {#if k == 'snowflake_oauth'} + {/if}
{/if} {/if} {/each} -
-
- - -
-
- -
- {:else if tab === 'oauth'} -
- When one of the below options is set, you will be able to create a specific resource - containing a token automatically generated by the third-party provider. -
- To test it after setting an oauth client, go to the Resources menu and create a new one of the - type of your oauth client (i.e. a 'github' resource if you set Github OAuth). -
Learn more
-
-
- -
- -
- {#each Object.keys(oauths) as k} - {#if oauths[k] && !('login_config' in oauths[k])} - {#if !['slack', 'teams'].includes(k) && oauths[k]} -
-
- - - { - delete oauths[k] - oauths = { ...oauths } - }} - /> -
-
- - - {#if k === 'azure_oauth'} - - {:else if !windmillBuiltins.includes(k) && k != 'slack'} - - {/if} - {#if k == 'snowflake_oauth'} - - {/if} -
-
+
+ + {#if oauth_name == 'custom'} + + {:else} + {/if} - {/if} - {/each} - -
- - {#if oauth_name == 'custom'} - - {:else} - - {/if} - -
- {:else if tab == 'scim'} - + +
+ {:else if tab == 'scim'} + {@render scim?.()} + {/if} {/if} diff --git a/frontend/src/lib/components/AutoscalingConfigEditor.svelte b/frontend/src/lib/components/AutoscalingConfigEditor.svelte index e90f39a9ef..9afa7a004f 100644 --- a/frontend/src/lib/components/AutoscalingConfigEditor.svelte +++ b/frontend/src/lib/components/AutoscalingConfigEditor.svelte @@ -11,11 +11,15 @@ import Label from './Label.svelte' import MultiSelect from 'svelte-multiselect' - export let config: AutoscalingConfig | undefined - export let worker_tags: string[] | undefined + interface Props { + config: AutoscalingConfig | undefined + worker_tags: string[] | undefined + } + + let { config = $bindable(), worker_tags }: Props = $props() const dispatch = createEventDispatcher() - let test_input: number = 3 + let test_input: number = $state(3)
@@ -49,11 +53,11 @@ } }} /> - + - +
{/if}
{#each result['render_all'] as res} - { forceJson = ev.detail === 'json' }} > - {#if ['table-col', 'table-row', 'table-row-object'].includes(resultKind ?? '')} - - {:else} - - {/if} - + {#snippet children({ item })} + {#if ['table-col', 'table-row', 'table-row-object'].includes(resultKind ?? '')} + + {:else} + + {/if} + + {/snippet} {/if}
{#if customUi?.disableAiFix !== true} - + {@render copilot_fix?.()} {/if} {#if toolbarLocation === 'self'} @@ -647,7 +677,7 @@ >{JSON.stringify(result.error.extra, null, 4)} {/if} - + {@render children?.()}
{#if !isTest && language === 'bun'}
@@ -723,7 +753,7 @@ /> - - {#if has_draft} {#if draft_only} - Never deployed and is only a draft + {#snippet text()} + Never deployed and is only a draft + {/snippet} Draft only {:else} - Is deployed and has a draft + {#snippet text()} + Is deployed and has a draft + {/snippet} +Draft {/if} diff --git a/frontend/src/lib/components/DropdownV2.svelte b/frontend/src/lib/components/DropdownV2.svelte index 6843c12927..a41ef9e50e 100644 --- a/frontend/src/lib/components/DropdownV2.svelte +++ b/frontend/src/lib/components/DropdownV2.svelte @@ -16,7 +16,10 @@ import ResolveOpen from '$lib/components/common/menu/ResolveOpen.svelte' import Button from '$lib/components/common/button/Button.svelte' import { twMerge } from 'tailwind-merge' + import TriggerableByAI from './TriggerableByAI.svelte' + export let aiId: string | undefined = undefined + export let aiDescription: string | undefined = undefined export let items: Item[] | (() => Item[]) | (() => Promise) = [] export let disabled = false export let placement: Placement = 'bottom-end' @@ -28,6 +31,8 @@ export let customWidth: number | undefined = undefined export let customMenu = false + let buttonEl: HTMLButtonElement | undefined = undefined + const { elements: { menu, item, trigger }, states, @@ -76,7 +81,9 @@ + buttonEl?.click()} /> - + {/snippet} {/if} @@ -429,7 +483,7 @@ class="rounded-full p-1 text-gray-500 bg-white duration-200 hover:bg-gray-600 focus:bg-gray-600 hover:text-white dark:bg-gray-700 dark:text-white dark:hover:bg-gray-800" aria-label="Clear" - on:click={() => { + onclick={() => { dispatch('delete', argName) }} > @@ -461,103 +515,103 @@ {isFlowInput} {isAppInput} on:change={() => { - schema = schema + schema = $state.snapshot(schema) dispatch('change', schema) }} > - + {#snippet typeeditor()} {#if isFlowInput || isAppInput} {/if} - + {/snippet} {#if isFlowInput || isAppInput} { - schema = schema + schema = $state.snapshot(schema) dispatch('change', schema) dispatch('schemaChange') }} @@ -651,7 +705,9 @@ bind:this={itemPicker} pickCallback={(path, _) => { if (pickForField) { - args[pickForField] = '$var:' + path + if (args) { + args[pickForField] = '$var:' + path + } } }} itemName="Variable" @@ -664,17 +720,19 @@ ...x }))} > -
- -
+ {#snippet submission()} +
+ +
+ {/snippet} diff --git a/frontend/src/lib/components/Editor.svelte b/frontend/src/lib/components/Editor.svelte index 82db22bf99..08ce87d57e 100644 --- a/frontend/src/lib/components/Editor.svelte +++ b/frontend/src/lib/components/Editor.svelte @@ -153,7 +153,7 @@ import { AIChatEditorHandler } from './copilot/chat/monaco-adapter' import GlobalReviewButtons from './copilot/chat/GlobalReviewButtons.svelte' import { writable } from 'svelte/store' - import { formatResourceTypes } from './copilot/chat/core' + import { formatResourceTypes } from './copilot/chat/script/core' import FakeMonacoPlaceHolder from './FakeMonacoPlaceHolder.svelte' // import EditorTheme from './EditorTheme.svelte' @@ -187,7 +187,6 @@ export let files: Record | undefined = {} export let extraLib: string | undefined = undefined export let changeTimeout: number = 500 - export let isAiPanelOpen: boolean = false export let loadAsync = false let lang = scriptLangToEditorLang(scriptLang) @@ -207,7 +206,7 @@ let websocketInterval: NodeJS.Timeout | undefined let lastWsAttempt: Date = new Date() let nbWsAttempt = 0 - let disposeMethod: () => void | undefined + let disposeMethod: (() => void) | undefined const dispatch = createEventDispatcher() let graphqlService: MonacoGraphQLAPI | undefined = undefined @@ -1268,15 +1267,20 @@ onFileChanges() - editor = meditor.create(divEl as HTMLDivElement, { - ...editorConfig(code, lang, automaticLayout, fixedOverflowWidgets), - model, - fontSize: !small ? 14 : 12, - lineNumbersMinChars, - // overflowWidgetsDomNode: widgets, - tabSize: lang == 'python' ? 4 : 2, - folding - }) + try { + editor = meditor.create(divEl as HTMLDivElement, { + ...editorConfig(code, lang, automaticLayout, fixedOverflowWidgets), + model, + fontSize: !small ? 14 : 12, + lineNumbersMinChars, + // overflowWidgetsDomNode: widgets, + tabSize: lang == 'python' ? 4 : 2, + folding + }) + } catch (e) { + console.error('Error loading monaco:', e) + return + } keepModelAroundToAvoidDisposalOfWorkers() @@ -1333,9 +1337,6 @@ startLine: selection.startLineNumber, endLine: selection.endLineNumber }) - if (!isAiPanelOpen) { - dispatch('toggleAiPanel') - } } else { dispatch('toggleAiPanel') } diff --git a/frontend/src/lib/components/EditorBar.svelte b/frontend/src/lib/components/EditorBar.svelte index 14fe08e45e..5b5681b3b3 100644 --- a/frontend/src/lib/components/EditorBar.svelte +++ b/frontend/src/lib/components/EditorBar.svelte @@ -1,8 +1,10 @@ - - +
{#if schema && mod} @@ -77,10 +81,10 @@ selected ? 'bg-surface-selected' : 'hover:bg-surface-hover' )} disabled={!schema} - on:click={handleClick} + onclick={handleClick} > -
- {#if mod.value.type === 'rawscript'} - - {:else if mod.value.type === 'script'} - {#if mod.value.path.startsWith('hub/')} -
- -
- {:else} - + {#snippet icon()} +
+ {#if mod.value.type === 'rawscript'} + + {:else if mod.value.type === 'script'} + {#if mod.value.path.startsWith('hub/')} +
+ +
+ {:else} + + {/if} {/if} - {/if} -
+
+ {/snippet}
{:else} diff --git a/frontend/src/lib/components/FlowBuilder.svelte b/frontend/src/lib/components/FlowBuilder.svelte index fe88ba24be..bff1b5dac5 100644 --- a/frontend/src/lib/components/FlowBuilder.svelte +++ b/frontend/src/lib/components/FlowBuilder.svelte @@ -1,23 +1,18 @@ - + - +{@render children?.()} {#if $pathStore} {/if} - { - applyCopilotFlowInputs() - finishStepGen() - }} - on:canceled={async () => { - clearFlowInputsFromStep($copilotModulesStore[0]?.id) - finishStepGen() - }} - bind:open={openCopilotInputsModal} - inputs={Object.keys(copilotFlowInputs)} - />
@@ -1319,25 +861,21 @@
- {#if $copilotCurrentStepStore !== undefined} -
- {/if}
{ - const currentModules = $flowStore?.value?.modules + const currentModules = flowStore.val?.value?.modules - $flowStore = undo(history, $flowStore) + flowStore.val = undo(history, flowStore.val) - const newModules = $flowStore?.value?.modules + const newModules = flowStore.val?.value?.modules const restoredModules = newModules?.filter( (node) => !currentModules?.some((currentNode) => currentNode?.id === node?.id) ) @@ -1355,7 +893,7 @@ $selectedIdStore = 'Input' }} on:redo={() => { - $flowStore = redo(history) + flowStore.val = redo(history) }} />
@@ -1388,7 +926,7 @@
{/if} - {#if !disableAi && customUi?.topBar?.aiBuilder != false} - + {#if !disableAi && customUi?.topBar?.aiBuilder != false && !aiChatManager.open} + aiChatManager.openChat()} /> {/if} { @@ -1521,7 +1057,7 @@ } }} on:testWithArgs={(e) => { - $previewArgsStore = JSON.parse(JSON.stringify(e.detail)) + previewArgsStore.val = JSON.parse(JSON.stringify(e.detail)) flowPreviewButtons?.openPreview(true) }} {savedFlow} diff --git a/frontend/src/lib/components/FlowGraphViewer.svelte b/frontend/src/lib/components/FlowGraphViewer.svelte index f69c1c9450..5297ed3584 100644 --- a/frontend/src/lib/components/FlowGraphViewer.svelte +++ b/frontend/src/lib/components/FlowGraphViewer.svelte @@ -45,8 +45,7 @@ modules={flow?.value?.modules} failureModule={flow?.value?.failure_module} preprocessorModule={flow?.value?.preprocessor_module} - on:select={(e) => { - let nodeId = e?.detail + onSelect={(nodeId) => { if (nodeId === 'triggers') { dispatch('triggerDetail') return @@ -55,7 +54,7 @@ } else if (nodeId === 'preprocessor') { stepDetail = flow?.value?.preprocessor_module } else { - stepDetail = dfs(flow?.value?.modules ?? [], (m) => m).find((m) => m?.id === e?.detail) + stepDetail = dfs(flow?.value?.modules ?? [], (m) => m).find((m) => m?.id === nodeId) } stepDetail = stepDetail ?? nodeId dispatch('select', stepDetail) diff --git a/frontend/src/lib/components/FlowLoopIterationPreview.svelte b/frontend/src/lib/components/FlowLoopIterationPreview.svelte index 6cc235e242..61b3f27662 100644 --- a/frontend/src/lib/components/FlowLoopIterationPreview.svelte +++ b/frontend/src/lib/components/FlowLoopIterationPreview.svelte @@ -11,13 +11,23 @@ import { CornerDownLeft, Play, RefreshCw, X } from 'lucide-svelte' import type { Schema } from '$lib/common' - export let open: boolean + interface Props { + open: boolean + jobId?: string | undefined + job?: Job | undefined + modules: FlowModule[] + previewArgs?: Record + whileLoop?: boolean + } - export let jobId: string | undefined = undefined - export let job: Job | undefined = undefined - export let modules: FlowModule[] - export let previewArgs: Record = {} - export let whileLoop = false + let { + open, + jobId = $bindable(undefined), + job = $bindable(undefined), + modules, + previewArgs = $bindable({}), + whileLoop = false + }: Props = $props() export const forloopSchema: Schema = { $schema: 'https://json-schema.org/draft/2020-12/schema' as string | undefined, @@ -54,10 +64,10 @@ type: 'object' } - let selectedJobStep: string | undefined = undefined + let selectedJobStep: string | undefined = $state(undefined) - let isRunning: boolean = false - let jobProgressReset: () => void + let isRunning: boolean = $state(false) + let jobProgressReset: (() => void) | undefined = $state(undefined) export function test() { runPreview(previewArgs, undefined) @@ -70,7 +80,7 @@ args: Record, restartedFrom: RestartedFrom | undefined ) { - jobProgressReset() + jobProgressReset?.() const newFlow = { value: { modules }, summary: '' } jobId = await runFlowPreview(args, newFlow, $pathStore, restartedFrom) isRunning = true @@ -89,12 +99,14 @@ } } - $: if (job?.type === 'CompletedJob') { - isRunning = false - } + $effect(() => { + if (job?.type === 'CompletedJob') { + isRunning = false + } + }) - +
@@ -156,7 +168,7 @@ diff --git a/frontend/src/lib/components/FlowPreviewContent.svelte b/frontend/src/lib/components/FlowPreviewContent.svelte index 23efc93695..638b63877f 100644 --- a/frontend/src/lib/components/FlowPreviewContent.svelte +++ b/frontend/src/lib/components/FlowPreviewContent.svelte @@ -13,7 +13,7 @@ import { AlertTriangle, ArrowRight, CornerDownLeft, Play, RefreshCw, X } from 'lucide-svelte' import { emptyString, sendUserToast } from '$lib/utils' import { dfs } from './flows/dfs' - import { sliceModules } from './flows/flowStateUtils' + import { sliceModules } from './flows/flowStateUtils.svelte' import InputSelectedBadge from './schema/InputSelectedBadge.svelte' import Toggle from './Toggle.svelte' import JsonInputs from './JsonInputs.svelte' @@ -53,7 +53,7 @@ export function test() { renderCount++ - runPreview($previewArgs, undefined) + runPreview(previewArgs.val, undefined) } const { @@ -75,9 +75,9 @@ function extractFlow(previewMode: 'upTo' | 'whole'): OpenFlow { if (previewMode === 'whole') { - return $flowStore + return flowStore.val } else { - const flow: Flow = JSON.parse(JSON.stringify($flowStore)) + const flow: Flow = JSON.parse(JSON.stringify(flowStore.val)) const idOrders = dfs(flow.value.modules, (x) => x.id) let upToIndex = idOrders.indexOf($selectedId) @@ -97,13 +97,13 @@ initial = false } try { - lastPreviewFlow = JSON.stringify($flowStore) + lastPreviewFlow = JSON.stringify(flowStore.val) jobProgressReset() const newFlow = extractFlow(previewMode) jobId = await runFlowPreview(args, newFlow, $pathStore, restartedFrom) isRunning = true if (inputSelected) { - savedArgs = $previewArgs + savedArgs = previewArgs.val inputSelected = undefined } } catch (e) { @@ -120,7 +120,7 @@ case 'Enter': if (event.ctrlKey || event.metaKey) { event.preventDefault() - runPreview($previewArgs, undefined) + runPreview(previewArgs.val, undefined) } break @@ -156,20 +156,20 @@ } } - let savedArgs = $previewArgs + let savedArgs = previewArgs.val let inputSelected: 'captures' | 'history' | 'saved' | undefined = undefined async function selectInput(input, type?: 'captures' | 'history' | 'saved' | undefined) { if (!input) { - $previewArgs = savedArgs + previewArgs.val = savedArgs inputSelected = undefined setTimeout(() => { preventEscape = false }, 100) } else { - $previewArgs = input + previewArgs.val = input inputSelected = type preventEscape = true - jsonEditor?.setCode(JSON.stringify($previewArgs ?? {}, null, '\t')) + jsonEditor?.setCode(JSON.stringify(previewArgs.val ?? {}, null, '\t')) } } @@ -185,7 +185,7 @@ async function loadIndividualStepsStates() { // console.log('loadIndividualStepsStates') - dfs($flowStore.value.modules, async (module) => { + dfs(flowStore.val.value.modules, async (module) => { // console.log('module', $flowStateStore[module.id], module.id) const prev = $flowStateStore[module.id]?.previewResult if (prev && prev != NEVER_TESTED_THIS_FAR) { @@ -283,7 +283,7 @@ variant="border" title={`Re-start this flow from step ${selectedJobStep} (included).`} on:click={() => { - runPreview($previewArgs, { + runPreview(previewArgs.val, { flow_job_id: jobId, step_id: selectedJobStep, branch_or_iteration_n: 0 @@ -308,7 +308,7 @@ color="blue" startIcon={{ icon: RefreshCw }} on:click={() => { - runPreview($previewArgs, { + runPreview(previewArgs.val, { flow_job_id: jobId, step_id: selectedJobStep, branch_or_iteration_n: 0 @@ -354,7 +354,7 @@ btnClasses="!p-1 !w-[34px] !ml-1" aria-label="Restart flow" on:click|once={() => { - runPreview($previewArgs, { + runPreview(previewArgs.val, { flow_job_id: jobId, step_id: selectedJobStep, branch_or_iteration_n: branchOrIterationN @@ -375,7 +375,7 @@ color="dark" size="sm" btnClasses="w-full max-w-lg" - on:click={() => runPreview($previewArgs, undefined)} + on:click={() => runPreview(previewArgs.val, undefined)} id="flow-editor-test-flow-drawer" shortCut={{ Icon: CornerDownLeft }} > @@ -385,7 +385,7 @@ {/if}
- {#if lastPreviewFlow && JSON.stringify($flowStore) != lastPreviewFlow} + {#if lastPreviewFlow && JSON.stringify(flowStore.val) != lastPreviewFlow}
{ selectInput(e.detail.payload, e.detail?.type) @@ -419,7 +419,7 @@ >
schemaFormWithArgPicker?.resetSelected()} + onReject={() => schemaFormWithArgPicker?.resetSelected()} {inputSelected} />
@@ -433,7 +433,7 @@ }} lightMode on:change={(e) => { - jsonEditor?.setCode(JSON.stringify($previewArgs ?? {}, null, '\t')) + jsonEditor?.setCode(JSON.stringify(previewArgs.val ?? {}, null, '\t')) refresh() }} /> @@ -445,7 +445,7 @@ bind:this={jsonEditor} on:select={(e) => { if (e.detail) { - $previewArgs = e.detail + previewArgs.val = e.detail } }} updateOnBlur={false} @@ -458,10 +458,10 @@ { - savedArgs = $previewArgs + savedArgs = previewArgs.val }} bind:isValid /> diff --git a/frontend/src/lib/components/FlowStatusViewerInner.svelte b/frontend/src/lib/components/FlowStatusViewerInner.svelte index 8a2a630268..a4a6d9edcd 100644 --- a/frontend/src/lib/components/FlowStatusViewerInner.svelte +++ b/frontend/src/lib/components/FlowStatusViewerInner.svelte @@ -29,7 +29,7 @@ import Alert from './common/alert/Alert.svelte' import FlowGraphViewerStep from './FlowGraphViewerStep.svelte' import FlowGraphV2 from './graph/FlowGraphV2.svelte' - import { buildPrefix } from './graph/graphBuilder' + import { buildPrefix } from './graph/graphBuilder.svelte' const dispatch = createEventDispatcher() @@ -1253,31 +1253,29 @@ success={jobId != undefined && isSuccess(job?.['success'])} flowModuleStates={$localModuleStates} bind:expandedSubflows - on:select={(e) => { + onSelect={(e) => { + console.log('onSelect', e) if (rightColumnSelect != 'node_definition') { rightColumnSelect = 'node_status' } - if (typeof e.detail == 'string') { - if (e.detail == 'Input') { + if (typeof e == 'string') { + if (e == 'Input') { selectedNode = 'start' stepDetail = undefined - } else if (e.detail == 'Result') { + } else if (e == 'Result') { selectedNode = 'end' stepDetail = 'end' } else { - const mod = dfs(job?.raw_flow?.modules ?? [], (m) => m).find( - (m) => m?.id === e?.detail - ) + const mod = dfs(job?.raw_flow?.modules ?? [], (m) => m).find((m) => m?.id === e) stepDetail = mod - selectedNode = e?.detail + selectedNode = e } } else { - stepDetail = e.detail - selectedNode = e.detail.id + stepDetail = e + selectedNode = e.id } }} - on:selectedIteration={async (e) => { - let detail = e.detail + onSelectedIteration={async (detail) => { if (detail.manuallySet) { let rootJobId = detail.id await tick() diff --git a/frontend/src/lib/components/FlowViewer.svelte b/frontend/src/lib/components/FlowViewer.svelte index 49ad0bc54b..4f2ab8a456 100644 --- a/frontend/src/lib/components/FlowViewer.svelte +++ b/frontend/src/lib/components/FlowViewer.svelte @@ -8,20 +8,30 @@ import FlowViewerInner from './FlowViewerInner.svelte' import FlowInputViewer from './FlowInputViewer.svelte' - export let flow: { - summary: string - description?: string - value: FlowValue - schema?: any + interface Props { + flow: { + summary: string + description?: string + value: FlowValue + schema?: any + } + initialOpen?: number | undefined + noSide?: boolean + noGraph?: boolean + tab?: 'ui' | 'raw' | 'schema' + noSummary?: boolean + noGraphDownload?: boolean } - export let initialOpen: number | undefined = undefined - export let noSide = false - export let noGraph: boolean = false - - export let tab: 'ui' | 'raw' | 'schema' = noGraph ? 'schema' : 'ui' - export let noSummary = false - export let noGraphDownload = false + let { + flow, + initialOpen = undefined, + noSide = false, + noGraph = false, + tab = $bindable(noGraph ? 'schema' : 'ui'), + noSummary = false, + noGraphDownload = false + }: Props = $props() let open: { [id: number]: boolean } = {} if (initialOpen) { @@ -38,7 +48,7 @@ Raw Input Schema - + {#snippet content()}
{#if !noSummary} @@ -65,5 +75,5 @@
- + {/snippet} diff --git a/frontend/src/lib/components/FlowViewerInner.svelte b/frontend/src/lib/components/FlowViewerInner.svelte index e4bec76006..0e300c8410 100644 --- a/frontend/src/lib/components/FlowViewerInner.svelte +++ b/frontend/src/lib/components/FlowViewerInner.svelte @@ -11,16 +11,20 @@ import HighlightTheme from './HighlightTheme.svelte' import { filteredContentForExport } from './flows/utils' - export let flow: { - summary: string - description?: string - value: FlowValue - schema?: any + interface Props { + flow: { + summary: string + description?: string + value: FlowValue + schema?: any + } } - $: flowFiltered = filteredContentForExport(flow) + let { flow }: Props = $props() - let rawType: 'json' | 'yaml' = 'yaml' + let flowFiltered = $derived(filteredContentForExport(flow)) + + let rawType: 'json' | 'yaml' = $state('yaml') function trimStringToLines(inputString: string, maxLines: number = 100): string { const lines = inputString?.split('\n') ?? [] @@ -29,7 +33,7 @@ return linesToKeep.join('\n') } - let code: string = '' + let code: string = $state('') function computeCode() { const str = @@ -44,11 +48,13 @@ code = str } - let shouldDisplayLoadMore = false + let shouldDisplayLoadMore = $state(false) - $: flowFiltered && rawType && computeCode() + $effect(() => { + flowFiltered && rawType && computeCode() + }) - let maxLines = 100 + let maxLines = $state(100) @@ -62,7 +68,7 @@ > YAML JSON - + {#snippet content()}
{/if}
-
+ {/snippet}
diff --git a/frontend/src/lib/components/FolderEditor.svelte b/frontend/src/lib/components/FolderEditor.svelte index 22229d8f8e..5ae36a7e24 100644 --- a/frontend/src/lib/components/FolderEditor.svelte +++ b/frontend/src/lib/components/FolderEditor.svelte @@ -7,7 +7,6 @@ GranularAclService, GroupService } from '$lib/gen' - import AutoComplete from 'simple-svelte-autocomplete' import TableCustom from './TableCustom.svelte' import { Alert, Button, Drawer, DrawerContent } from './common' import Skeleton from './common/skeleton/Skeleton.svelte' @@ -19,6 +18,7 @@ import Label from './Label.svelte' import { sendUserToast } from '$lib/toast' import { createEventDispatcher } from 'svelte' + import Select from './Select.svelte' export let name: string let can_write = false @@ -184,25 +184,20 @@
- (ownerItem = '')} - let:item - > - - + (ownerItem = '')}> + {#snippet children({ item })} + + + {/snippet}
{#key ownerKind} - !perms?.map((y) => y.owner_name).includes('u/' + x)) : groups.filter((x) => !perms?.map((y) => y.owner_name).includes('g/' + x))} - bind:selectedItem={ownerItem} - /> + + +
+ {:else} + + {/if} + + + + + + + + + +
+ +
diff --git a/frontend/src/lib/components/GroupEditor.svelte b/frontend/src/lib/components/GroupEditor.svelte index fbe03bd9f0..2d7a1ce889 100644 --- a/frontend/src/lib/components/GroupEditor.svelte +++ b/frontend/src/lib/components/GroupEditor.svelte @@ -7,7 +7,6 @@ type InstanceGroup } from '$lib/gen' import { userStore, workspaceStore } from '$lib/stores' - import AutoComplete from 'simple-svelte-autocomplete' import { createEventDispatcher } from 'svelte' import { Button } from './common' import Skeleton from './common/skeleton/Skeleton.svelte' @@ -18,6 +17,7 @@ import ToggleButton from './common/toggleButton-v2/ToggleButton.svelte' import Section from './Section.svelte' import Label from './Label.svelte' + import Select from './Select.svelte' export let name: string let can_write = false @@ -129,7 +129,7 @@
{#if can_write}
- + { - navigator.clipboard.writeText(token) - sendUserToast('Copied to clipboard') + on:click|preventDefault|stopPropagation={() => { + if (token) { + navigator.clipboard.writeText(token) + sendUserToast('Copied to clipboard') + } }} - placeholder="Select tags to generate a jwt token" + placeholder="Select tags to generate a JWT token" type="text" disabled value={token} - class="pr-8 text-sm text-secondary" + class="w-full pr-10 pl-3 py-2 text-sm text-gray-600 bg-gray-50 border border-gray-300 rounded-lg cursor-pointer hover:bg-gray-100 transition truncate" /> + +
-
- Pass the env variables: -
    -
  • MODE=agent
  • -
  • AGENT_TOKEN={'""'}
  • -
  • BASE_INTERNAL_URL={'""'}
  • +
    + Set the following environment variables: +
      +
    • MODE=agent
    • +
    • AGENT_TOKEN=<token>
    • +
    • BASE_INTERNAL_URL=<base url>
    - to a worker to have it act as an HTTP agent worker. INIT_SCRIPT, if needed, must be passed as an - env variable. - -

    - Remember to have at least one normal worker that listens to the tags `flow` and `dependency` - (or `flow-$workspace` and `dependency-$workspace` if using workspace specific default tags) - to have flow and dependency job being runnable as agent workers can't run dependency jobs - nor can run the flow state machine (but can run the subjobs within them). +

    + to a worker to have it act as an HTTP agent worker. + INIT_SCRIPT, if needed, must be passed as an env variable.

    + + Ensure at least one normal worker is running and listening to the tags + flow and dependency + (or flow-<workspace> and dependency-<workspace> if + using workspace-specific default tags), because agent workers + cannot run dependency jobs + nor execute the + flow state machine. They can, however, run subjobs within flows. + + +
    + Use the following API endpoint with a superadmin bearer token: + POST /api/agent_workers/create_agent_token +
    +	{`
    +	  "worker_group": "agent",
    +	  "tags": ["tag1", "tag2"],
    +	  "exp": 1717334400
    +	`}
    +					
    + The JSON response will contain the generated JWT token. +
    +
    - - -
    - Use the following api endpoint with a superadmin bearer token: - POST /api/agent_workers/create_agent_token - with body: -
    -			
    -{`{
    -	"worker_group": "agent",
    -	"tags": ["tag1", "tag2"],
    -	"exp": 1717334400 // 3 years from now
    -}`}
    -					
    -				
    - JSON response will be the JWT token. -
    -
diff --git a/frontend/src/lib/components/IconedResourceType.svelte b/frontend/src/lib/components/IconedResourceType.svelte index 5daea14d17..7375c64cd7 100644 --- a/frontend/src/lib/components/IconedResourceType.svelte +++ b/frontend/src/lib/components/IconedResourceType.svelte @@ -1,18 +1,35 @@
@@ -20,12 +37,13 @@ {name} {/if} {#if iconComponent} + {@const SvelteComponent = iconComponent} - + {:else if formatExtension} - + {:else} @@ -34,4 +52,3 @@ {name} {/if}
- diff --git a/frontend/src/lib/components/InfiniteList.svelte b/frontend/src/lib/components/InfiniteList.svelte index 6457d5ea60..b4afaf41ba 100644 --- a/frontend/src/lib/components/InfiniteList.svelte +++ b/frontend/src/lib/components/InfiniteList.svelte @@ -6,26 +6,48 @@ const dispatch = createEventDispatcher() - export let loading = false - export let items: any[] | undefined = undefined - export let selectedItemId: any | undefined = undefined - export let isEmpty: boolean = true - export let length: number = 0 - export let rounded: boolean = true - export let noBorder: boolean = false - export let extraRowClasses: { bgSelected: string; bgHover: string; class: string } = { - bgSelected: '', - bgHover: '', - class: '' + interface Props { + loading?: boolean + items?: any[] | undefined + selectedItemId?: any | undefined + isEmpty?: boolean + length?: number + rounded?: boolean + noBorder?: boolean + extraRowClasses?: { bgSelected: string; bgHover: string; class: string } + neverShowLoader?: boolean + columns?: import('svelte').Snippet + extra_row?: import('svelte').Snippet<[any]> + children?: import('svelte').Snippet<[any]> + empty?: import('svelte').Snippet<[any]> } - export let neverShowLoader = false + + let { + loading = $bindable(false), + items = $bindable(undefined), + selectedItemId = $bindable(undefined), + isEmpty = $bindable(true), + length = $bindable(0), + rounded = true, + noBorder = false, + extraRowClasses = { + bgSelected: '', + bgHover: '', + class: '' + }, + neverShowLoader = false, + columns, + extra_row, + children, + empty + }: Props = $props() const perPage = 20 - let hasMore = false + let hasMore = $state(false) let page = 1 let hasAlreadyFailed = false - let hovered: any | undefined = undefined + let hovered: any | undefined = $state(undefined) let initLoad = false let loadInputs: ((page: number, perPage: number) => Promise) | undefined = undefined let deleteItemFn: ((id: any) => Promise) | undefined = undefined @@ -39,7 +61,7 @@ page = 1 } - let loadingMore = false + let loadingMore = $state(false) export async function loadData(loadOption: 'refresh' | 'forceRefresh' | 'loadMore' = 'loadMore') { // console.log('loadData', loadOption, length, items?.length) @@ -146,7 +168,7 @@ {noBorder} {neverShowLoader} > - + {@render columns?.()} (hovered = e.detail ? 'extraRow' : undefined)} > - + {@render extra_row?.({ hover: hovered === 'extraRow' })} {#each items ?? [] as item, index} {@const hover = item.id === hovered} @@ -171,16 +193,16 @@ )} on:hover={(e) => (hovered = e.detail ? item.id : undefined)} > - + {@render children?.({ item, hover })} {/each} - + {#snippet emptyMessage()} {#if (!items || items?.length === 0) && (!loading || neverShowLoader)} - + {@render empty?.({ items })} {/if} - + {/snippet} diff --git a/frontend/src/lib/components/WorkerTagSelect.svelte b/frontend/src/lib/components/WorkerTagSelect.svelte index 8cac3f0a64..616cb6d809 100644 --- a/frontend/src/lib/components/WorkerTagSelect.svelte +++ b/frontend/src/lib/components/WorkerTagSelect.svelte @@ -2,10 +2,8 @@ import { workerTags, workspaceStore } from '$lib/stores' import { WorkerService } from '$lib/gen' - import Select from './apps/svelte-select/lib/index' import { createEventDispatcher } from 'svelte' - import DarkModeObserver from './DarkModeObserver.svelte' - import { SELECT_INPUT_DEFAULT_STYLE } from '$lib/defaults' + import Select from './Select.svelte' let { tag = $bindable(), @@ -32,57 +30,18 @@ ...(tag && tag != '' && !($workerTags ?? []).includes(tag) ? [tag] : []), ...($workerTags ?? []) ]) - let darkMode: boolean = $state(false) - -
{#if !noLabel}
tag
{/if} { - selectedDay = e.detail.value - outputs.day.set(Number(selectedDay)) - }} - on:clear={() => { - selectedDay = '' - outputs.day.set(undefined) - }} + bind:value={ + () => selectedDay, + (v) => { + selectedDay = v ?? '' + outputs.day.set(v ? Number(v) : undefined) + } + } items={Array.from({ length: computeDayPerMonth(selectedMonth, selectedYear) }, (_, i) => { return { label: String(i + 1), value: String(i + 1) } })} class={twMerge('text-clip min-w-0', css?.input?.class, 'wm-date-select')} - containerStyles={(darkMode - ? SELECT_INPUT_DEFAULT_STYLE.containerStylesDark - : SELECT_INPUT_DEFAULT_STYLE.containerStyles) + css?.input?.style} - inputStyles={SELECT_INPUT_DEFAULT_STYLE.inputStyles} + containerStyle={css?.input?.style} placeholder="Pick a day" + clearable />
{/if} @@ -263,22 +253,17 @@ class={twMerge('grow', resolvedConfig?.orientation === 'horizontal' ? 'w-1/2' : 'w-full')} > { - selectedYear = e.detail.value - outputs.year.set(Number(selectedYear)) - }} - on:clear={() => { - selectedYear = '' - outputs.year.set(undefined) - }} - items={Array.from({ length: 201 }, (_, i) => `${1900 + i}`)} + bind:value={ + () => selectedYear, + (v) => { + selectedYear = v ?? '' + outputs.year.set(selectedYear ? Number(selectedYear) : undefined) + } + } + items={Array.from({ length: 201 }, (_, i) => `${1900 + i}`).map((value) => ({ + value + }))} placeholder="Pick a year" - inputStyles={SELECT_INPUT_DEFAULT_STYLE.inputStyles} class={twMerge('text-clip min-w-0', css?.input?.class, 'wm-date-select')} - containerStyles={(darkMode - ? SELECT_INPUT_DEFAULT_STYLE.containerStylesDark - : SELECT_INPUT_DEFAULT_STYLE.containerStyles) + css?.input?.style} + containerStyle={css?.input?.style} clearable />
diff --git a/frontend/src/lib/components/apps/components/inputs/AppMultiSelect.svelte b/frontend/src/lib/components/apps/components/inputs/AppMultiSelect.svelte index 24b02c41c7..d586fd85f3 100644 --- a/frontend/src/lib/components/apps/components/inputs/AppMultiSelect.svelte +++ b/frontend/src/lib/components/apps/components/inputs/AppMultiSelect.svelte @@ -1,206 +1,23 @@ -{#each Object.keys(components['multiselectcomponent'].initialData.configuration) as key (key)} - -{/each} - -{#each Object.keys(css ?? {}) as key (key)} - -{/each} -
{ - $selectedComponent = [id] - - if (!e.shiftKey) { - e.stopPropagation() - } - selectedComponent.set([id]) - }} - use:floatingRef - bind:clientWidth={w} + >This version of the multiselect component is not supported anymore. Recreate the multiselect + component to use this new version.
- {#if !value || Array.isArray(value)} - { - if (event?.type === 'removeAll') { - outputs?.result.set([]) - } else { - outputs?.result.set([...(value ?? [])]) - } - }} - onopen={() => { - $selectedComponent = [id] - open = true - }} - onclose={() => { - open = false - }} - > - {#snippet children({ option })} - - -
{ - let newe = new MouseEvent('mouseup') - e.target?.['parentElement']?.dispatchEvent(newe) - }} - > - {option} -
- {/snippet} -
- - - - {:else} - Value {value} is not an array - {/if} -
diff --git a/frontend/src/lib/components/apps/components/inputs/AppSelect.svelte b/frontend/src/lib/components/apps/components/inputs/AppSelect.svelte index 6d7abcaf55..5c9fff9c35 100644 --- a/frontend/src/lib/components/apps/components/inputs/AppSelect.svelte +++ b/frontend/src/lib/components/apps/components/inputs/AppSelect.svelte @@ -1,6 +1,5 @@ {#each Object.keys(components['selectcomponent'].initialData.configuration) as key (key)} @@ -242,7 +258,7 @@
{ + onpointerdown={(e) => { if (!e.shiftKey) { e.stopPropagation() } @@ -250,7 +266,7 @@ > {#if Array.isArray(listItems) && listItems.every((x) => x && typeof x == 'object' && typeof x['label'] == 'string' && `value` in x)} {#if resolvedConfig.nativeHtmlSelect} - {#if resolvedConfig.placeholder} {/if} @@ -260,34 +276,22 @@ {:else} + clearable + onFocus={() => !$connectingInput.opened && ($selectedComponent = [id])} + /> {/if} {:else} @@ -298,16 +302,18 @@ >
- -
- - The selectable items should be an array of {'{"value": any, "label":}'}. Found: -
{JSON.stringify(listItems, null, 4)}
-
-
-
+ {#snippet text()} + +
+ + The selectable items should be an array of {'{"value": any, "label":}'}. Found: +
{JSON.stringify(listItems, null, 4)}
+
+
+
+ {/snippet} {/if}
diff --git a/frontend/src/lib/components/apps/components/inputs/AppSelectTab.svelte b/frontend/src/lib/components/apps/components/inputs/AppSelectTab.svelte index 32e4e74bc6..39069b2fb5 100644 --- a/frontend/src/lib/components/apps/components/inputs/AppSelectTab.svelte +++ b/frontend/src/lib/components/apps/components/inputs/AppSelectTab.svelte @@ -1,5 +1,5 @@ {#each Object.keys(components['selecttabcomponent'].initialData.configuration) as key (key)} diff --git a/frontend/src/lib/components/apps/components/layout/AppConditionalWrapper.svelte b/frontend/src/lib/components/apps/components/layout/AppConditionalWrapper.svelte index 915a1963b6..d3618b6bed 100644 --- a/frontend/src/lib/components/apps/components/layout/AppConditionalWrapper.svelte +++ b/frontend/src/lib/components/apps/components/layout/AppConditionalWrapper.svelte @@ -1,5 +1,5 @@ diff --git a/frontend/src/lib/components/apps/components/layout/AppDecisionTree.svelte b/frontend/src/lib/components/apps/components/layout/AppDecisionTree.svelte index f9637204fc..c29bb46825 100644 --- a/frontend/src/lib/components/apps/components/layout/AppDecisionTree.svelte +++ b/frontend/src/lib/components/apps/components/layout/AppDecisionTree.svelte @@ -1,5 +1,5 @@ {#if Object.keys(resolvedConditions).length === nodes.length} diff --git a/frontend/src/lib/components/apps/components/layout/AppDrawer.svelte b/frontend/src/lib/components/apps/components/layout/AppDrawer.svelte index 4e83813db0..679bb928c8 100644 --- a/frontend/src/lib/components/apps/components/layout/AppDrawer.svelte +++ b/frontend/src/lib/components/apps/components/layout/AppDrawer.svelte @@ -1,4 +1,6 @@ {#each Object.keys(components['drawercomponent'].initialData.configuration) as key (key)} @@ -105,7 +122,7 @@ parentComponentId: id, subGridIndex: 0 } - appDrawer.toggleDrawer() + appDrawer?.toggleDrawer() }} size={resolvedConfig.size} color={resolvedConfig.color} @@ -122,7 +139,6 @@ {#if everRender} $runnableComponents?.[id]?.cb?.map((cb) => cb?.())) }} + disableChatOffset={true} > - { - appDrawer?.toggleDrawer() - $focusedGrid = undefined - }} - fullScreen={$mode !== 'dnd'} - > -
{ - e?.stopPropagation() - if (!$connectingInput.opened) { - $selectedComponent = [id] - $focusedGrid = { - parentComponentId: id, - subGridIndex: 0 - } - } + {#snippet children({ open })} + { + appDrawer?.toggleDrawer() + $focusedGrid = undefined }} + fullScreen={$mode !== 'dnd'} > - {#if $app.subgrids?.[`${id}-0`]} - { - if (!$connectingInput.opened) { - $selectedComponent = [id] - $focusedGrid = { - parentComponentId: id, - subGridIndex: 0 - } +
{ + e?.stopPropagation() + if (!$connectingInput.opened) { + $selectedComponent = [id] + $focusedGrid = { + parentComponentId: id, + subGridIndex: 0 } - }} - /> - {/if} -
-
+ } + }} + > + {#if $app.subgrids?.[`${id}-0`]} + { + if (!$connectingInput.opened) { + $selectedComponent = [id] + $focusedGrid = { + parentComponentId: id, + subGridIndex: 0 + } + } + }} + /> + {/if} +
+
+ {/snippet}
{:else if $app.subgrids?.[`${id}-0`]} diff --git a/frontend/src/lib/components/apps/components/layout/AppList.svelte b/frontend/src/lib/components/apps/components/layout/AppList.svelte index 6498410036..ef2b906178 100644 --- a/frontend/src/lib/components/apps/components/layout/AppList.svelte +++ b/frontend/src/lib/components/apps/components/layout/AppList.svelte @@ -15,19 +15,32 @@ import { twMerge } from 'tailwind-merge' import ResolveStyle from '../helpers/ResolveStyle.svelte' - export let id: string - export let componentInput: AppInput | undefined - export let configuration: RichConfigurations - export let customCss: ComponentCustomCSS<'listcomponent'> | undefined = undefined - export let render: boolean - export let initializing: boolean | undefined + interface Props { + id: string + componentInput: AppInput | undefined + configuration: RichConfigurations + customCss?: ComponentCustomCSS<'listcomponent'> | undefined + render: boolean + initializing: boolean | undefined + } + + let { + id, + componentInput, + configuration, + customCss = undefined, + render, + initializing = $bindable() + }: Props = $props() const { app, focusedGrid, selectedComponent, worldStore, connectingInput, allIdsInPath, mode } = getContext('AppViewerContext') - let page = 0 + let page = $state(0) - let everRender = render - $: render && !everRender && (everRender = true) + let everRender = $state(render) + $effect.pre(() => { + render && !everRender && (everRender = true) + }) const outputs = initOutput($worldStore, id, { result: undefined, @@ -36,9 +49,8 @@ page: 0 }) - let resolvedConfig = initConfig( - components['listcomponent'].initialData.configuration, - configuration + let resolvedConfig = $state( + initConfig(components['listcomponent'].initialData.configuration, configuration) ) function onFocus() { @@ -48,20 +60,22 @@ } } - let css = initCss($app.css?.listcomponent, customCss) - let result: any[] | undefined = undefined + let css = $state(initCss($app.css?.listcomponent, customCss)) + let result: any[] | undefined = $state(undefined) - $: isCard = resolvedConfig.width?.selected == 'card' + let isCard = $derived(resolvedConfig.width?.selected == 'card') - let inputs = {} - let loading: boolean = false - let isPreviousLoading = false - let isNextLoading = false + let inputs = $state({}) + let loading: boolean = $state(false) + let isPreviousLoading = $state(false) + let isNextLoading = $state(false) - $: if (!loading) { - isPreviousLoading = false - isNextLoading = false - } + $effect.pre(() => { + if (!loading) { + isPreviousLoading = false + isNextLoading = false + } + }) function getPagination( configuration: { @@ -99,11 +113,13 @@ } } - $: pagination = getPagination( - resolvedConfig.pagination?.configuration, - resolvedConfig.pagination?.selected, - result, - page + let pagination = $derived( + getPagination( + resolvedConfig.pagination?.configuration, + resolvedConfig.pagination?.selected, + result, + page + ) ) @@ -150,8 +166,8 @@ : 'overflow-auto'} {isCard ? 'gap-2 flex-wrap' : resolvedConfig?.displayBorders - ? 'divide-y flex-col' - : 'flex-col'}" + ? 'divide-y flex-col' + : 'flex-col'}" > {#if $app.subgrids?.[`${id}-0`] && Array.isArray(result) && result.length > 0} {#each result ?? [] as value, index (index)} @@ -162,16 +178,16 @@ isCard ? `min-width: ${resolvedConfig.width?.configuration?.card?.minWidthPx}px; ` : '' - } max-height: ${resolvedConfig.heightPx}px;` + } max-height: ${resolvedConfig.heightPx}px;` : ''} class={inRange ? `${ $allIdsInPath.includes(id) ? 'overflow-visible' : resolvedConfig.heightPx - ? 'overflow-auto' - : '' - } ${!isCard ? 'w-full' : resolvedConfig?.displayBorders ? 'border' : ''}` + ? 'overflow-auto' + : '' + } ${!isCard ? 'w-full' : resolvedConfig?.displayBorders ? 'border' : ''}` : 'h-0 float overflow-hidden invisible absolute'} > + import { stopPropagation } from 'svelte/legacy' + import { getContext } from 'svelte' import SubGridEditor from '../../editor/SubGridEditor.svelte' import type { AppViewerContext, ComponentCustomCSS, RichConfigurations } from '../../types' @@ -17,16 +19,29 @@ import Disposable from '$lib/components/common/drawer/Disposable.svelte' import AlignWrapper from '../helpers/AlignWrapper.svelte' - export let customCss: ComponentCustomCSS<'modalcomponent'> | undefined = undefined - export let id: string - export let configuration: RichConfigurations - export let horizontalAlignment: 'left' | 'center' | 'right' | undefined = undefined - export let verticalAlignment: 'top' | 'center' | 'bottom' | undefined = undefined - export let noWFull = false - export let render: boolean + interface Props { + customCss?: ComponentCustomCSS<'modalcomponent'> | undefined + id: string + configuration: RichConfigurations + horizontalAlignment?: 'left' | 'center' | 'right' | undefined + verticalAlignment?: 'top' | 'center' | 'bottom' | undefined + noWFull?: boolean + render: boolean + onOpenRecomputeIds?: string[] | undefined + onCloseRecomputeIds?: string[] | undefined + } - export let onOpenRecomputeIds: string[] | undefined = undefined - export let onCloseRecomputeIds: string[] | undefined = undefined + let { + customCss = undefined, + id, + configuration, + horizontalAlignment = undefined, + verticalAlignment = undefined, + noWFull = false, + render, + onOpenRecomputeIds = undefined, + onCloseRecomputeIds = undefined + }: Props = $props() const { app, @@ -40,23 +55,24 @@ breakpoint } = getContext('AppViewerContext') - let everRender = render - $: render && !everRender && (everRender = true) + let everRender = $state(render) + $effect.pre(() => { + render && !everRender && (everRender = true) + }) //used so that we can count number of outputs setup for first refresh const outputs = initOutput($worldStore, id, { open: false }) - let css = initCss($app.css?.modalcomponent, customCss) - let disposable: Disposable | undefined = undefined + let css = $state(initCss($app.css?.modalcomponent, customCss)) + let disposable: Disposable | undefined = $state(undefined) - let resolvedConfig = initConfig( - components['modalcomponent'].initialData.configuration, - configuration + let resolvedConfig = $state( + initConfig(components['modalcomponent'].initialData.configuration, configuration) ) - let unclickableOutside = false + let unclickableOutside = $state(false) function unclosableModal() { unclickableOutside = true setTimeout(() => { @@ -81,14 +97,16 @@ disposable?.closeDrawer() } } - let wrapperHeight: number = 0 - let headerHeight: number = 0 + let wrapperHeight: number = $state(0) + let headerHeight: number = $state(0) - $: containerHeight = Math.min( - // 8px * 2 of padding - maxHeight($app.subgrids?.[`${id}-0`] ?? [], 0, $breakpoint) * (ROW_HEIGHT + ROW_GAP_Y) + 16, - // 32px (2rem) of top and bottom margin - wrapperHeight - headerHeight - 64 + let containerHeight = $derived( + Math.min( + // 8px * 2 of padding + maxHeight($app.subgrids?.[`${id}-0`] ?? [], 0, $breakpoint) * (ROW_HEIGHT + ROW_GAP_Y) + 16, + // 32px (2rem) of top and bottom margin + wrapperHeight - headerHeight - 64 + ) ) async function getMenuElements(): Promise { @@ -154,9 +172,6 @@ { outputs?.open.set(true) @@ -167,80 +182,82 @@ onCloseRecomputeIds?.forEach((id) => $runnableComponents?.[id]?.cb?.map((cb) => cb?.())) }} > -
+ {#snippet children({ handleClickAway, zIndex, open })}
{ - if ($mode !== 'dnd' && !unclickableOutside) { - handleClickAway(e) - } - }} + class={twMerge( + `${ + $mode == 'dnd' ? 'absolute' : 'fixed' + } top-0 bottom-0 left-0 right-0 transition-all duration-50`, + open ? ' bg-black bg-opacity-60' : 'h-0 overflow-hidden invisible' + )} + style="z-index: {zIndex}" + bind:clientHeight={wrapperHeight} >
-
{resolvedConfig.modalTitle}
-
- -
-
- -
{ - e?.stopPropagation() - if (!$connectingInput.opened) { - $selectedComponent = [id] - $focusedGrid = { - parentComponentId: id, - subGridIndex: 0 + style={css?.popup?.style} + class={twMerge('mx-24 mt-8 bg-surface wm-modal rounded-lg relative', css?.popup?.class)} + use:clickOutside={{ + capture: false, + stopPropagation: false, + exclude: getMenuElements, + onClickOutside: (e: MouseEvent) => { + if ($mode !== 'dnd' && !unclickableOutside) { + handleClickAway(e) } } }} > - {#if $app.subgrids?.[`${id}-0`]} - { - if (!$connectingInput.opened) { - $selectedComponent = [id] - $focusedGrid = { - parentComponentId: id, - subGridIndex: 0 - } +
+
{resolvedConfig.modalTitle}
+
+ +
+
+ +
{ + e?.stopPropagation() + if (!$connectingInput.opened) { + $selectedComponent = [id] + $focusedGrid = { + parentComponentId: id, + subGridIndex: 0 } - }} - /> - {/if} + } + }} + > + {#if $app.subgrids?.[`${id}-0`]} + { + if (!$connectingInput.opened) { + $selectedComponent = [id] + $focusedGrid = { + parentComponentId: id, + subGridIndex: 0 + } + } + }} + /> + {/if} +
-
+ {/snippet}
{:else if $app.subgrids?.[`${id}-0`]} diff --git a/frontend/src/lib/components/apps/components/layout/AppSplitpanes.svelte b/frontend/src/lib/components/apps/components/layout/AppSplitpanes.svelte index 90dbf14618..4424280ced 100644 --- a/frontend/src/lib/components/apps/components/layout/AppSplitpanes.svelte +++ b/frontend/src/lib/components/apps/components/layout/AppSplitpanes.svelte @@ -1,4 +1,6 @@ {#each Object.keys(css ?? {}) as key (key)} @@ -87,20 +102,20 @@ {#if everRender} -
+
{#key sumedup} {#each sumedup as paneSize, index (index)}
{ + onpointerdown={stopPropagation(() => { $selectedComponent = [id] $focusedGrid = { parentComponentId: id, subGridIndex: index } - }} + })} > {#if $app.subgrids?.[`${id}-${index}`]} - import { getContext } from 'svelte' + import { getContext, untrack } from 'svelte' import { initOutput } from '../../editor/appUtils' import SubGridEditor from '../../editor/SubGridEditor.svelte' import type { AppViewerContext, ComponentCustomCSS } from '../../types' @@ -14,16 +14,31 @@ import RunnableComponent from '../helpers/RunnableComponent.svelte' import RunnableWrapper from '../helpers/RunnableWrapper.svelte' - export let id: string - export let componentContainerHeight: number - export let tabs: string[] - export let customCss: ComponentCustomCSS<'steppercomponent'> | undefined = undefined - export let render: boolean - export let recomputeIds: string[] | undefined = undefined - export let extraQueryParams: Record = {} - export let componentInput: AppInput | undefined - export let onNext: string[] | undefined = undefined - export let onPrevious: string[] | undefined = undefined + interface Props { + id: string + componentContainerHeight: number + tabs: string[] + customCss?: ComponentCustomCSS<'steppercomponent'> | undefined + render: boolean + recomputeIds?: string[] | undefined + extraQueryParams?: Record + componentInput: AppInput | undefined + onNext?: string[] | undefined + onPrevious?: string[] | undefined + } + + let { + id, + componentContainerHeight, + tabs, + customCss = undefined, + render, + recomputeIds = undefined, + extraQueryParams = {}, + componentInput, + onNext = undefined, + onPrevious = undefined + }: Props = $props() const { app, @@ -36,16 +51,18 @@ runnableComponents } = getContext('AppViewerContext') - let everRender = render - $: render && !everRender && (everRender = true) + let everRender = $state(render) + $effect.pre(() => { + render && !everRender && (everRender = true) + }) - let selected = tabs[0] - let tabHeight: number = 0 - let footerHeight: number = 0 - let runnableComponent: RunnableComponent - let selectedIndex = tabs?.indexOf(selected) ?? -1 - let maxReachedIndex = -1 - let statusByStep = [] as Array<'success' | 'error' | 'pending'> + let selected = $state(tabs[0]) + let tabHeight: number = $state(0) + let footerHeight: number = $state(0) + let runnableComponent: RunnableComponent | undefined = $state() + let selectedIndex = $state(tabs?.indexOf(selected) ?? -1) + let maxReachedIndex = $state(-1) + let statusByStep = $state([] as Array<'success' | 'error' | 'pending'>) let debugMode: boolean = false let outputs = initOutput($worldStore, id, { @@ -74,7 +91,8 @@ } } - let result: { error: { name: string; message: string; stack: string } } | undefined = undefined + let result: { error: { name: string; message: string; stack: string } } | undefined = + $state(undefined) async function runStep(targetIndex: number) { statusByStep[selectedIndex] = 'pending' @@ -122,14 +140,22 @@ } handleTabSelection() + }, + setSelectedIndex(index) { + if (index >= 0 && index < tabs.length) { + selected = tabs[index] + handleTabSelection() + } } } - $: selected != undefined && handleTabSelection() - let css = initCss($app.css?.steppercomponent, customCss) - $: lastStep = selectedIndex === tabs.length - 1 + $effect.pre(() => { + selected != undefined && untrack(() => handleTabSelection()) + }) + let css = $state(initCss($app.css?.steppercomponent, customCss)) + let lastStep = $derived(selectedIndex === tabs.length - 1) - let directionClicked: 'left' | 'right' | undefined = undefined + let directionClicked: 'left' | 'right' | undefined = $state(undefined) {#each Object.keys(css ?? {}) as key (key)} diff --git a/frontend/src/lib/components/apps/components/layout/AppTabs.svelte b/frontend/src/lib/components/apps/components/layout/AppTabs.svelte index 977ccc69d9..e9d028a5ed 100644 --- a/frontend/src/lib/components/apps/components/layout/AppTabs.svelte +++ b/frontend/src/lib/components/apps/components/layout/AppTabs.svelte @@ -1,6 +1,9 @@ @@ -153,8 +203,8 @@ > {#each tabs ?? [] as res, index} - + {#snippet text()}
Hand Mode @@ -1003,7 +1058,7 @@ esc
-
+ {/snippet}
- +
{ + onscroll={parseScroll} + onmouseenter={() => { mouseOverGridView = true }} - on:mouseleave={() => { + onmouseleave={() => { mouseOverGridView = false }} class={classNames( @@ -1125,7 +1180,9 @@
- Component library + {#snippet text()} + Component library + {/snippet} - Component settings + {#snippet text()} + Component settings + {/snippet} - Global styling + {#snippet text()} + Global styling + {/snippet} hideRightPanel()} direction="right" />
-
- - {#if $selectedComponent !== undefined} - { - befSelected = undefined - selectedTab = 'insert' - }} - /> - - {:else} -
- Select a component to see the settings for it -
- {/if} -
- - - - - - -
+ {#snippet content()} +
+ + {#if $selectedComponent !== undefined} + { + befSelected = undefined + selectedTab = 'insert' + }} + /> + + {:else} +
+ Select a component to see the settings for it +
+ {/if} +
+ + + + + + +
+ {/snippet}
@@ -1232,22 +1295,23 @@ ...x }))} > -
- -
+ +
+ {/snippet} diff --git a/frontend/src/lib/components/apps/editor/AppEditorHeader.svelte b/frontend/src/lib/components/apps/editor/AppEditorHeader.svelte index 48bbfc0288..42d6f7a0f3 100644 --- a/frontend/src/lib/components/apps/editor/AppEditorHeader.svelte +++ b/frontend/src/lib/components/apps/editor/AppEditorHeader.svelte @@ -1,4 +1,7 @@ - + -{#if $$slots.unsavedConfirmationModal} - { +{#if unsavedConfirmationModal} + {@render unsavedConfirmationModal?.({ + diffDrawer, + additionalExitAction: () => { setTheme(priorDarkMode) - }} - getInitialAndModifiedValues={() => ({ + }, + getInitialAndModifiedValues: () => ({ savedValue: savedApp, modifiedValue: { summary: $summary, @@ -929,8 +961,8 @@ policy, custom_path: customPath } - })} - /> + }) + })} {/if}

Summary

- + { + onkeyup={() => { if ($appPath == '' && $summary?.length > 0 && !dirtyPath) { path?.setName( $summary @@ -989,15 +1021,17 @@ />
-
- -
+ {#snippet actions()} +
+ +
+ {/snippet} {/if} @@ -1028,15 +1062,15 @@ {/if} Summary
- + { + onkeydown={stopPropagation(bubble('keydown'))} + onkeyup={() => { if ($appPath == '' && $summary?.length > 0 && !dirtyPath) { path?.setName( $summary @@ -1052,7 +1086,7 @@
Deployment message
- + -
- + - -
+ }} + > + Deploy + +
+ {/snippet}
{#if $appPath == ''} @@ -1219,7 +1255,7 @@ class={customPathError === '' ? '' : 'border border-red-700 bg-red-100 border-opacity-30 focus:border-red-700 focus:border-opacity-30 focus-visible:ring-red-700 focus-visible:ring-opacity-25 focus-visible:border-red-700'} - on:input={() => { + oninput={() => { dirtyCustomPath = true }} /> @@ -1282,10 +1318,16 @@ undoProps={{ disabled: $history?.index === 0 }} redoProps={{ disabled: $history && $history?.index === $history.history.length - 1 }} on:undo={() => { - $app = undo(history, $app) + const napp = undo(history, $app) + for (const key in napp) { + $app[key] = napp[key] + } }} on:redo={() => { - $app = redo(history) + const napp = redo(history) + for (const key in napp) { + $app[key] = napp[key] + } }} /> @@ -1296,22 +1338,23 @@ on:selected={({ detail }) => { $app.fullscreen = detail === 'true' }} - let:item > - - + {#snippet children({ item })} + + + {/snippet} {/if} {#if $app} @@ -1322,60 +1365,63 @@ setTheme(theme) }} selected={$app.darkMode === undefined ? 'auto' : $app.darkMode ? 'dark' : 'sun'} - let:item > - - - + {#snippet children({ item })} + + + + {/snippet} {/if}
- - - - {#if $breakpoint === 'sm'} - + {#snippet children({ item })} + - {/if} + + {#if $breakpoint === 'sm'} + + {/if} + {/snippet}
diff --git a/frontend/src/lib/components/apps/editor/AppExportButton.svelte b/frontend/src/lib/components/apps/editor/AppExportButton.svelte index b44ba8b547..9094980535 100644 --- a/frontend/src/lib/components/apps/editor/AppExportButton.svelte +++ b/frontend/src/lib/components/apps/editor/AppExportButton.svelte @@ -34,7 +34,7 @@ YAML JSON - + {#snippet content()}
-
+ {/snippet}
diff --git a/frontend/src/lib/components/apps/editor/AppJobsDrawer.svelte b/frontend/src/lib/components/apps/editor/AppJobsDrawer.svelte index aef77f3f51..9f6bdb8d19 100644 --- a/frontend/src/lib/components/apps/editor/AppJobsDrawer.svelte +++ b/frontend/src/lib/components/apps/editor/AppJobsDrawer.svelte @@ -78,8 +78,8 @@ ? 'bg-red-600 !border-blue-600' : 'bg-red-400' : selectedJobId == id - ? 'text-blue-600' - : '' + ? 'text-blue-600' + : '' )} on:click={() => { selectedJobId = id @@ -266,7 +266,7 @@
- + {#snippet actions()} {#if refreshComponents} {/if} - + {/snippet} diff --git a/frontend/src/lib/components/apps/editor/AppJsonEditor.svelte b/frontend/src/lib/components/apps/editor/AppJsonEditor.svelte index 86c3123daa..e52eb6cff8 100644 --- a/frontend/src/lib/components/apps/editor/AppJsonEditor.svelte +++ b/frontend/src/lib/components/apps/editor/AppJsonEditor.svelte @@ -85,7 +85,7 @@ {/if} - + {#snippet actions()} {#if !$userStore?.operator} {/if} - + {/snippet} diff --git a/frontend/src/lib/components/apps/editor/AppReportsDrawerInner.svelte b/frontend/src/lib/components/apps/editor/AppReportsDrawerInner.svelte index 6850c90b58..576fac8643 100644 --- a/frontend/src/lib/components/apps/editor/AppReportsDrawerInner.svelte +++ b/frontend/src/lib/components/apps/editor/AppReportsDrawerInner.svelte @@ -24,47 +24,54 @@ import { CUSTOM_TAGS_SETTING, WORKSPACE_SLACK_BOT_TOKEN_PATH } from '$lib/consts' import { loadSchemaFromPath } from '$lib/infer' import { hubPaths } from '$lib/hub' - export let appPath: string - export let open = false + import { untrack } from 'svelte' + interface Props { + appPath: string + open?: boolean + } - let appReportingEnabled = false - let appReportingStartupDuration = 5 + let { appPath, open = $bindable(false) }: Props = $props() + + let appReportingEnabled = $state(false) + let appReportingStartupDuration = $state(5) let appReportingSchedule: { cron: string timezone: string - } = { + } = $state({ cron: '0 0 12 * *', timezone: Intl.DateTimeFormat().resolvedOptions().timeZone - } - let selectedTab: 'email' | 'slack' | 'discord' | 'custom' = $enterpriseLicense - ? 'slack' - : 'custom' + }) + let selectedTab: 'email' | 'slack' | 'discord' | 'custom' = $state( + $enterpriseLicense ? 'slack' : 'custom' + ) - let screenshotKind: 'pdf' | 'png' = 'pdf' + let screenshotKind: 'pdf' | 'png' = $state('pdf') - let customPath: string | undefined = undefined - let customPathSchema: Record = {} - let args: Record = {} - let areArgsValid = true - let customWidth: null | number = 1200 - let customHeight: null | number = null + let customPath: string | undefined = $state(undefined) + let customPathSchema: Record = $state({}) + let args: Record = $state({}) + let areArgsValid = $state(true) + let customWidth: null | number = $state(1200) + let customHeight: null | number = $state(null) - $: customPath - ? loadSchemaFromPath(customPath).then((schema) => { - customPathSchema = schema - ? { - ...schema, - properties: Object.fromEntries( - Object.entries(schema.properties ?? {}).filter( - ([key, _]) => key !== 'screenshot' && key !== 'app_path' && key !== 'kind' + $effect(() => { + customPath + ? loadSchemaFromPath(customPath).then((schema) => { + customPathSchema = schema + ? { + ...schema, + properties: Object.fromEntries( + Object.entries(schema.properties ?? {}).filter( + ([key, _]) => key !== 'screenshot' && key !== 'app_path' && key !== 'kind' + ) ) - ) - } - : {} - }) - : (customPathSchema = {}) + } + : {} + }) + : (customPathSchema = {}) + }) - let isSlackConnectedWorkspace = false + let isSlackConnectedWorkspace = $state(false) async function getWorspaceSlackSetting() { const settings = await WorkspaceService.getSettings({ workspace: $workspaceStore! @@ -120,7 +127,7 @@ ) : {} setTimeout(() => { - args = structuredClone(nargs) + args = structuredClone($state.snapshot(nargs)) }) customPath = @@ -134,7 +141,9 @@ } catch (err) {} } - $: appPath && getAppReportingInfo() + $effect(() => { + appPath && untrack(() => getAppReportingInfo()) + }) async function disableAppReporting() { const flowPath = appPath + '_reports' @@ -496,7 +505,7 @@ export async function main( appReportingEnabled = true } - let testLoading = false + let testLoading = $state(false) async function testReport() { try { testLoading = true @@ -545,12 +554,14 @@ export async function main( } } - let disabled = true - $: disabled = - emptyString(appReportingSchedule.cron) || - (selectedTab === 'custom' && emptyString(customPath)) || - (selectedTab === 'slack' && !isSlackConnectedWorkspace) || - !areArgsValid + let disabled = $state(true) + $effect(() => { + disabled = + emptyString(appReportingSchedule.cron) || + (selectedTab === 'custom' && emptyString(customPath)) || + (selectedTab === 'slack' && !isSlackConnectedWorkspace) || + !areArgsValid + }) +> + {#snippet actions()}
{appReportingEnabled ? 'Update' : 'Save and enable'} - + {/snippet}
Send a PDF or PNG preview of the app at a given schedule. Enabling this feature will create a diff --git a/frontend/src/lib/components/apps/editor/GridEditor.svelte b/frontend/src/lib/components/apps/editor/GridEditor.svelte index 5163a92885..01a725683b 100644 --- a/frontend/src/lib/components/apps/editor/GridEditor.svelte +++ b/frontend/src/lib/components/apps/editor/GridEditor.svelte @@ -1,5 +1,5 @@ - - + - - -
- - + + +
+ + {@render children?.()} {#if contextMenuVisible}
- + {#each menuItems as item} - - + + - {item.tooltip?.text} + {#snippet text()} + {item.tooltip?.text} {#if item.tooltip?.link}
@@ -192,8 +206,8 @@
- {/if}
+ {/if} + {/snippet} +
{/snippet} diff --git a/frontend/src/lib/components/apps/editor/component/ComponentNavigation.svelte b/frontend/src/lib/components/apps/editor/component/ComponentNavigation.svelte index fa0a8556d9..97d98bcaed 100644 --- a/frontend/src/lib/components/apps/editor/component/ComponentNavigation.svelte +++ b/frontend/src/lib/components/apps/editor/component/ComponentNavigation.svelte @@ -1,9 +1,34 @@ - - - + handlePaste(e, ctx)} /> diff --git a/frontend/src/lib/components/apps/editor/component/componentCallbacks.svelte.ts b/frontend/src/lib/components/apps/editor/component/componentCallbacks.svelte.ts new file mode 100644 index 0000000000..046d6bdd08 --- /dev/null +++ b/frontend/src/lib/components/apps/editor/component/componentCallbacks.svelte.ts @@ -0,0 +1,357 @@ +// import { getContext } from 'svelte' +import { + copyComponent, + findGridItem, + findGridItemParentGrid, + getAllSubgridsAndComponentIds, + insertNewGridItem +} from '../appUtils' +import type { AppEditorContext, AppViewerContext, GridItem } from '../../types' +import { push } from '$lib/history' +import { sendUserToast } from '$lib/toast' +import { gridColumns } from '../../gridUtils' +import { copyToClipboard } from '$lib/utils' +import { get } from 'svelte/store' + +// const { app, selectedComponent, focusedGrid, componentControl } = +// getContext('AppViewerContext') + +// const { history, movingcomponents, jobsDrawerOpen } = +// getContext('AppEditorContext') + +let tempGridItems: GridItem[] | undefined = undefined + +const ITEM_TYPE = 'wm-grid-items' + +function getSortedGridItemsOfChildren(ctx: { + app: AppViewerContext['app'] + focusedGrid: AppViewerContext['focusedGrid'] +}): GridItem[] { + const focusedGrid = get(ctx.focusedGrid) + if (!focusedGrid) { + return get(ctx.app).grid + } + + let subgrids = get(ctx.app).subgrids + if (!subgrids) { + return [] + } + + return subgrids[`${focusedGrid.parentComponentId}-${focusedGrid.subGridIndex}`] ?? [] +} + +function getGridItems(ctx: { + app: AppViewerContext['app'] + selectedComponent: AppViewerContext['selectedComponent'] +}): GridItem[] { + const app = get(ctx.app) + const selectedComponent = get(ctx.selectedComponent) + if (app.grid.find((item) => item.id === selectedComponent?.[0])) { + return get(ctx.app).grid + } + + if (!app.subgrids) { + return [] + } + + return ( + Object.values(app.subgrids ?? {}).find((grid) => + grid.find((item) => item.id === selectedComponent?.[0]) + ) ?? [] + ) +} + +export function left( + event: KeyboardEvent, + ctx: { + app: AppViewerContext['app'] + componentControl: AppViewerContext['componentControl'] + selectedComponent: AppViewerContext['selectedComponent'] + } +) { + const componentControl = get(ctx.componentControl) + const selectedComponent = get(ctx.selectedComponent) + if (!componentControl[selectedComponent?.[0] ?? '']?.left?.()) { + const sortedGridItems = getGridItems(ctx) + const currentIndex = sortedGridItems.findIndex( + (item) => item.id === (selectedComponent?.[0] ?? '') + ) + + if (currentIndex !== -1 && currentIndex > 0) { + const left = sortedGridItems[currentIndex - 1] + + if (left.data.type === 'tablecomponent' && left.data.actionButtons.length >= 1) { + ctx.selectedComponent.set([left.data.actionButtons[left.data.actionButtons.length - 1].id]) + } else if ( + (left.data.type === 'aggridcomponent' || + left.data.type === 'aggridcomponentee' || + left.data.type === 'dbexplorercomponent') && + Array.isArray(left.data.actions) && + left.data.actions.length >= 1 + ) { + ctx.selectedComponent.set([left.data.actions[left.data.actions.length - 1].id]) + } else { + ctx.selectedComponent.set([left.id]) + } + } + } + + event.preventDefault() +} + +export function right( + event: KeyboardEvent, + ctx: { + app: AppViewerContext['app'] + componentControl: AppViewerContext['componentControl'] + selectedComponent: AppViewerContext['selectedComponent'] + } +) { + const componentControl = get(ctx.componentControl) + const selectedComponent = get(ctx.selectedComponent) + let r = componentControl[selectedComponent?.[0] ?? '']?.right?.() + + if (typeof r === 'string') { + ctx.selectedComponent.set([r]) + r = componentControl[r]?.right?.(true) + } + + if (!r) { + const sortedGridItems = getGridItems(ctx) + const currentIndex = sortedGridItems.findIndex( + (item) => item.id === (selectedComponent?.[0] ?? '') + ) + + if (currentIndex !== -1 && currentIndex < sortedGridItems.length - 1) { + ctx.selectedComponent.set([sortedGridItems[currentIndex + 1].id]) + } + } + + event.preventDefault() +} + +export function down( + event: KeyboardEvent, + ctx: { + app: AppViewerContext['app'] + componentControl: AppViewerContext['componentControl'] + selectedComponent: AppViewerContext['selectedComponent'] + focusedGrid: AppViewerContext['focusedGrid'] + } +) { + const focusedGrid = get(ctx.focusedGrid) + const selectedComponent = get(ctx.selectedComponent) + const app = get(ctx.app) + if (!focusedGrid) { + ctx.selectedComponent.set([getSortedGridItemsOfChildren(ctx)[0]?.id]) + event.preventDefault() + } else if (app.subgrids) { + const index = focusedGrid?.subGridIndex ?? 0 + const subgrid = app.subgrids[`${selectedComponent}-${index}`] + + if (!subgrid || subgrid.length === 0) { + return + } + + if (subgrid) { + ctx.selectedComponent.set([subgrid[0].id]) + } + event.preventDefault() + } +} + +export function handleEscape( + event: KeyboardEvent, + ctx: { + selectedComponent: AppViewerContext['selectedComponent'] + focusedGrid: AppViewerContext['focusedGrid'] + } +) { + ctx.selectedComponent.set(undefined) + ctx.focusedGrid.set(undefined) + event.preventDefault() +} + +export function handleArrowUp( + event: KeyboardEvent, + ctx: { + app: AppViewerContext['app'] + selectedComponent: AppViewerContext['selectedComponent'] + focusedGrid: AppViewerContext['focusedGrid'] + } +) { + const selectedComponent = get(ctx.selectedComponent) + if (!selectedComponent) return + let parentId = findGridItemParentGrid(get(ctx.app), selectedComponent?.[0])?.split('-')[0] + + if (parentId) { + ctx.selectedComponent.set([parentId]) + } else { + ctx.selectedComponent.set(undefined) + ctx.focusedGrid.set(undefined) + } +} + +export async function handleCopy( + event: KeyboardEvent, + ctx: { + app: AppViewerContext['app'] + selectedComponent: AppViewerContext['selectedComponent'] + jobsDrawerOpen: AppEditorContext['jobsDrawerOpen'] + } +) { + const selectedComponent = get(ctx.selectedComponent) + if (!selectedComponent || get(ctx.jobsDrawerOpen) || window.getSelection()?.toString() != '') { + return + } + tempGridItems = undefined + const copiedGridItems = selectedComponent + .map((x) => findGridItem(get(ctx.app), x)) + .filter((x) => x != undefined) as GridItem[] + + copyGridItemsToClipboard(copiedGridItems, ctx, 'copy') +} + +async function copyGridItemsToClipboard( + items: GridItem[], + ctx: { + app: AppViewerContext['app'] + }, + type: 'copy' | 'cut' | undefined = undefined +) { + const app = get(ctx.app) + let allSubgrids = {} + for (let item of items) { + let subgrids = getAllSubgridsAndComponentIds(app, item.data)[0] + for (let key of subgrids) { + allSubgrids[key] = app.subgrids?.[key] + } + } + let success = await copyToClipboard( + JSON.stringify({ type: ITEM_TYPE, items, subgrids: allSubgrids }), + false + ) + + if (!success) { + sendUserToast('Could not copy to clipboard. Are you in an unsecure context?', true) + } else if (type) { + const text = type == 'copy' ? 'Copied' : 'Cut' + sendUserToast(`${text} ${items.length} component${items.length > 1 ? 's' : ''}`) + } +} + +export function handleCut( + event: KeyboardEvent, + ctx: { + selectedComponent: AppViewerContext['selectedComponent'] + history: AppEditorContext['history'] + app: AppViewerContext['app'] + movingcomponents: AppEditorContext['movingcomponents'] + } +) { + const selectedComponent = get(ctx.selectedComponent) + const app = get(ctx.app) + if (!selectedComponent) { + return + } + ctx.movingcomponents.set(JSON.parse(JSON.stringify(selectedComponent))) + push(ctx.history, app) + + let gridItems = selectedComponent + .map((x) => findGridItem(app, x)) + .filter((x) => x != undefined) as GridItem[] + copyGridItemsToClipboard(gridItems, ctx, 'cut') + + if (!gridItems) { + return + } + + // Store the grid item in a temp variable so we can paste it later + tempGridItems = gridItems +} + +export async function handlePaste( + event: ClipboardEvent, + ctx: { + app: AppViewerContext['app'] + selectedComponent: AppViewerContext['selectedComponent'] + focusedGrid: AppViewerContext['focusedGrid'] + componentControl: AppViewerContext['componentControl'] + history: AppEditorContext['history'] + jobsDrawerOpen: AppEditorContext['jobsDrawerOpen'] + movingcomponents: AppEditorContext['movingcomponents'] + } +) { + let classes = event.target?.['className'] + if ( + (typeof classes === 'string' && classes.includes('inputarea')) || + ['INPUT', 'TEXTAREA'].includes(document.activeElement?.tagName!) + ) { + return + } + event.preventDefault() + const app = get(ctx.app) + const focusedGrid = get(ctx.focusedGrid) + + push(ctx.history, app) + ctx.movingcomponents.set(undefined) + let copiedGridItems: GridItem[] | undefined = undefined + let subgrids = app.subgrids ?? {} + const txt = event?.clipboardData?.getData('text') + if (txt) { + try { + const parsed = JSON.parse(txt) + if (parsed.type == ITEM_TYPE) { + copiedGridItems = parsed.items + subgrids = parsed.subgrids ?? subgrids + } else { + copiedGridItems = undefined + } + } catch {} + } + + if (tempGridItems != undefined) { + for (let tempGridItem of tempGridItems) { + if ( + focusedGrid && + getAllSubgridsAndComponentIds(app, tempGridItem.data)[0].includes( + `${focusedGrid.parentComponentId}-${focusedGrid.subGridIndex}` + ) + ) { + sendUserToast('Cannot paste a component into itself', true) + return + } + let parentGrid = findGridItemParentGrid(app, tempGridItem.id) + if (parentGrid) { + app.subgrids && + (app.subgrids[parentGrid] = app.subgrids[parentGrid].filter( + (item) => item.id !== tempGridItem?.id + )) + } else { + app.grid = app.grid.filter((item) => item.id !== tempGridItem?.id) + } + + const gridItem = tempGridItem + insertNewGridItem( + app, + (id) => ({ ...gridItem.data, id }), + focusedGrid, + Object.fromEntries(gridColumns.map((column) => [column, gridItem[column]])), + tempGridItem.id + ) + } + copyGridItemsToClipboard(tempGridItems, ctx) + ctx.selectedComponent.set(tempGridItems.map((x) => x.id)) + + tempGridItems = undefined + } else if (copiedGridItems) { + let nitems: string[] = [] + for (let copiedGridItem of copiedGridItems) { + let newItem = copyComponent(app, copiedGridItem, focusedGrid, subgrids, []) + newItem && nitems.push(newItem) + } + ctx.selectedComponent.set(nitems.map((x) => x)) + } + + ctx.app.update((x) => x) +} diff --git a/frontend/src/lib/components/apps/editor/componentsPanel/ComponentList.svelte b/frontend/src/lib/components/apps/editor/componentsPanel/ComponentList.svelte index 4f3b353789..e597d9f057 100644 --- a/frontend/src/lib/components/apps/editor/componentsPanel/ComponentList.svelte +++ b/frontend/src/lib/components/apps/editor/componentsPanel/ComponentList.svelte @@ -233,12 +233,13 @@
{/if} -
- - - - - Selector - Comment - - - - {#each customisation.selectors as { selector, comment }} - - - {selector} - - - {#if comment} -
{comment}
- {/if} -
- - - -
- {/each} -
-
- - - - - Variable - Default value - Comment - - - - {#each customisation.variables as { variable, value, comment }} - - - {variable} - - - {value} - - - {#if comment} -
{comment}
- {/if} -
- - - -
- {/each} -
-
-
+ {#snippet content()} +
+ + + + + Selector + Comment + + + + {#each customisation.selectors as { selector, comment }} + + + {selector} + + + {#if comment} +
{comment}
+ {/if} +
+ + + +
+ {/each} +
+
+ + + + + Variable + Default value + Comment + + + + {#each customisation.variables as { variable, value, comment }} + + + {variable} + + + {value} + + + {#if comment} +
{comment}
+ {/if} +
+ + + +
+ {/each} +
+
+
+ {/snippet} {/each}
diff --git a/frontend/src/lib/components/apps/editor/componentsPanel/CssProperty.svelte b/frontend/src/lib/components/apps/editor/componentsPanel/CssProperty.svelte index 6ad1da6491..e8bff06d62 100644 --- a/frontend/src/lib/components/apps/editor/componentsPanel/CssProperty.svelte +++ b/frontend/src/lib/components/apps/editor/componentsPanel/CssProperty.svelte @@ -18,26 +18,47 @@ import ToggleButton from '$lib/components/common/toggleButton-v2/ToggleButton.svelte' import Popover from '$lib/components/Popover.svelte' import { tailwindClasses } from './tailwindUtils' - import { createDispatcherIfMounted } from '$lib/createDispatcherIfMounted' + import { deepEqual } from 'fast-equals' - export let name: string - export let value: ComponentCssProperty = {} - export let forceStyle: boolean = false - export let forceClass: boolean = false - export let quickStyleProperties: PropertyGroup[] | undefined = undefined - export let componentType: TypedComponent['type'] | undefined = undefined - export let tooltip: string | undefined = undefined - export let shouldDisplayLeft: boolean = false - export let shouldDisplayRight: boolean = false - export let overriden: boolean = false - export let overridding: boolean = false - export let wmClass: string | undefined = undefined + interface Props { + name: string + value?: ComponentCssProperty + forceStyle?: boolean + forceClass?: boolean + quickStyleProperties?: PropertyGroup[] | undefined + componentType?: TypedComponent['type'] | undefined + tooltip?: string | undefined + shouldDisplayLeft?: boolean + shouldDisplayRight?: boolean + overriden?: boolean + overridding?: boolean + wmClass?: string | undefined + } + + let { + name, + value = $bindable({}), + forceStyle = false, + forceClass = false, + quickStyleProperties = undefined, + componentType = undefined, + tooltip = undefined, + shouldDisplayLeft = false, + shouldDisplayRight = false, + overriden = false, + overridding = false, + wmClass = undefined + }: Props = $props() const dispatch = createEventDispatcher() - const dispatchIfMounted = createDispatcherIfMounted(dispatch) - let isQuickMenuOpen = false + let isQuickMenuOpen = $state(false) - $: dispatchIfMounted('change', value) + let prevValue = structuredClone($state.snapshot(value)) + $effect(() => { + if (deepEqual(prevValue, value)) return + prevValue = structuredClone($state.snapshot(value)) + dispatch('change', value) + }) function toggleQuickMenu() { try { @@ -52,10 +73,10 @@ } } - let richEditorOpen = false + let richEditorOpen = $state(false) - let dynamicClass: boolean = value?.evalClass !== undefined - let render = 0 + let dynamicClass: boolean = $state(value?.evalClass !== undefined) + let render = $state(0) {#key render} @@ -73,7 +94,9 @@ startIcon={{ icon: MoveLeft }} on:click={() => dispatch('left')} /> - {'Copy for this component'} + {#snippet text()} + {'Copy for this component'} + {/snippet} {/if} {#if shouldDisplayRight} @@ -85,9 +108,9 @@ startIcon={{ icon: MoveRight }} on:click={() => dispatch('right')} /> - + {#snippet text()} Copy for every {componentType ? ccomponents[componentType].name : 'component'} - + {/snippet} {/if} @@ -98,9 +121,9 @@ startIcon={{ icon: Copy }} on:click={() => copyToClipboard(wmClass)} /> - + {#snippet text()} Copy {wmClass} - + {/snippet}
@@ -113,7 +136,7 @@ {#if value.style !== undefined || forceStyle}
- +
Plain CSS
@@ -134,24 +157,25 @@ richEditorOpen = isQuickMenuOpen } }} - let:item > - - + {#snippet children({ item })} + + + {/snippet} {/if}
@@ -195,7 +219,7 @@ {/if} {#if value.class !== undefined || forceClass} - +
{#each unintitializedComponents as c} @@ -82,41 +83,39 @@ {#if !item}
Item {c} not found
{:else} - -
- - {c} - -
+ +
+ + {c} + +
-
- - {item?.data?.type || 'Unknown'} - -
+
+ + {item?.data?.type || 'Unknown'} + +
-
- Uninitialized -
-
- - -
+
+ Uninitialized +
+
+ +
{/if} - {/each}
@@ -166,7 +165,7 @@ on:click={() => { if ($app.subgrids && s) { delete $app.subgrids[s.subGridId] - $app = { ...$app } + $app = $app } }} > diff --git a/frontend/src/lib/components/apps/editor/contextPanel/LazyModePanel.svelte b/frontend/src/lib/components/apps/editor/contextPanel/LazyModePanel.svelte index b78debafee..534be983ac 100644 --- a/frontend/src/lib/components/apps/editor/contextPanel/LazyModePanel.svelte +++ b/frontend/src/lib/components/apps/editor/contextPanel/LazyModePanel.svelte @@ -9,13 +9,11 @@ import ToggleButton from '$lib/components/common/toggleButton-v2/ToggleButton.svelte' const { app } = getContext('AppViewerContext') - let code = JSON.stringify($app.lazyInitRequire) + let code = $state(JSON.stringify($app.lazyInitRequire)) - let selectedRendering = $app.eagerRendering - ? 'eager' - : $app.lazyInitRequire - ? 'lazy' - : 'semi-lazy' + let selectedRendering = $state( + $app.eagerRendering ? 'eager' : $app.lazyInitRequire ? 'lazy' : 'semi-lazy' + )
@@ -31,7 +29,6 @@ { if (e.detail == 'eager') { @@ -46,11 +43,12 @@ code = JSON.stringify($app.lazyInitRequire) } }} - let:item > - - - + {#snippet children({ item })} + + + + {/snippet} {#if selectedRendering == 'eager'}
diff --git a/frontend/src/lib/components/apps/editor/inlineScriptsPanel/InlineScriptEditor.svelte b/frontend/src/lib/components/apps/editor/inlineScriptsPanel/InlineScriptEditor.svelte index 3e280e9538..703aea891f 100644 --- a/frontend/src/lib/components/apps/editor/inlineScriptsPanel/InlineScriptEditor.svelte +++ b/frontend/src/lib/components/apps/editor/inlineScriptsPanel/InlineScriptEditor.svelte @@ -4,7 +4,7 @@ const bubble = createBubbler() import Button from '$lib/components/common/button/Button.svelte' import type { Preview } from '$lib/gen' - import { createEventDispatcher, getContext, onMount } from 'svelte' + import { createEventDispatcher, getContext, onMount, untrack } from 'svelte' import type { AppViewerContext, InlineScript } from '../../types' import { Maximize2, Trash2 } from 'lucide-svelte' import InlineScriptEditorDrawer from './InlineScriptEditorDrawer.svelte' @@ -196,6 +196,7 @@ $stateId++ } } + $app = $app } } @@ -231,7 +232,7 @@ $effect(() => { if (name && name !== lastName) { lastName = name - onNameChange() + untrack(() => onNameChange()) } }) diff --git a/frontend/src/lib/components/apps/editor/inlineScriptsPanel/InlineScriptEditorDrawer.svelte b/frontend/src/lib/components/apps/editor/inlineScriptsPanel/InlineScriptEditorDrawer.svelte index 41297ea965..23f072a3be 100644 --- a/frontend/src/lib/components/apps/editor/inlineScriptsPanel/InlineScriptEditorDrawer.svelte +++ b/frontend/src/lib/components/apps/editor/inlineScriptsPanel/InlineScriptEditorDrawer.svelte @@ -5,16 +5,28 @@ import type { InlineScript } from '../../types' import { Save } from 'lucide-svelte' - let scriptEditorDrawer: Drawer - export let appPath: string - export let inlineScript: InlineScript - export let editor: Editor | undefined = undefined - export let isOpen: boolean | undefined = undefined - export let id: string + let scriptEditorDrawer: Drawer | undefined = $state(undefined) + interface Props { + appPath: string + inlineScript: InlineScript + editor?: Editor | undefined + isOpen?: boolean | undefined + id: string + } + + let { + appPath, + inlineScript = $bindable(), + editor = undefined, + isOpen = $bindable(undefined), + id + }: Props = $props() export function openDrawer() { - scriptEditorDrawer.openDrawer?.() + scriptEditorDrawer?.openDrawer?.() } + + let args = $state({}) @@ -23,7 +35,7 @@ noPadding forceOverflowVisible on:close={() => { - scriptEditorDrawer.closeDrawer() + scriptEditorDrawer?.closeDrawer() editor?.setCode(inlineScript.content) }} > @@ -40,10 +52,11 @@ on:createScriptFromInlineScript tag={undefined} saveToWorkspace + {args} /> {/if} - + {#snippet actions()} - + {/snippet} diff --git a/frontend/src/lib/components/apps/editor/inlineScriptsPanel/InlineScriptRunnableByPath.svelte b/frontend/src/lib/components/apps/editor/inlineScriptsPanel/InlineScriptRunnableByPath.svelte index 866141b231..cfe33844a4 100644 --- a/frontend/src/lib/components/apps/editor/inlineScriptsPanel/InlineScriptRunnableByPath.svelte +++ b/frontend/src/lib/components/apps/editor/inlineScriptsPanel/InlineScriptRunnableByPath.svelte @@ -48,7 +48,6 @@ async function refreshScript(runnable: RunnableByPath) { try { let { schema } = await getScriptByPath(runnable.path) - console.log('schema1', schema) if (!deepEqual(runnable.schema, schema)) { runnable.schema = schema if (!runnable.schema.order) { diff --git a/frontend/src/lib/components/apps/editor/settingsPanel/AlignmentEditor.svelte b/frontend/src/lib/components/apps/editor/settingsPanel/AlignmentEditor.svelte index 4622a5a36c..eb28ce77b4 100644 --- a/frontend/src/lib/components/apps/editor/settingsPanel/AlignmentEditor.svelte +++ b/frontend/src/lib/components/apps/editor/settingsPanel/AlignmentEditor.svelte @@ -15,7 +15,11 @@ const { app } = getContext('AppViewerContext') - export let component: AppComponent + interface Props { + component: AppComponent + } + + let { component = $bindable() }: Props = $props() {#if component.horizontalAlignment || component.verticalAlignment} @@ -26,11 +30,12 @@ noWFull on:selected={() => ($app = $app)} bind:selected={component.horizontalAlignment} - let:item > - - - + {#snippet children({ item })} + + + + {/snippet} {/if} {#if component.type !== 'formcomponent' && component.verticalAlignment} @@ -38,11 +43,12 @@ noWFull on:selected={() => ($app = $app)} bind:selected={component.verticalAlignment} - let:item > - - - + {#snippet children({ item })} + + + + {/snippet} {/if}
diff --git a/frontend/src/lib/components/apps/editor/settingsPanel/CSSMigrationModal.svelte b/frontend/src/lib/components/apps/editor/settingsPanel/CSSMigrationModal.svelte index c114d37148..1955e636c2 100644 --- a/frontend/src/lib/components/apps/editor/settingsPanel/CSSMigrationModal.svelte +++ b/frontend/src/lib/components/apps/editor/settingsPanel/CSSMigrationModal.svelte @@ -15,6 +15,7 @@ import { sendUserToast } from '$lib/toast' import CloseButton from '$lib/components/common/CloseButton.svelte' import HighlightTheme from '$lib/components/HighlightTheme.svelte' + import Portal from '$lib/components/Portal.svelte' export let component: AppComponent | undefined const { app } = getContext('AppViewerContext') @@ -130,171 +131,171 @@ {#if migrationModalOpen} -