diff --git a/.cursor/rules/rust-best-practices.mdc b/.cursor/rules/rust-best-practices.mdc index e8511c67d4..bdbdb0d24d 100644 --- a/.cursor/rules/rust-best-practices.mdc +++ b/.cursor/rules/rust-best-practices.mdc @@ -47,6 +47,7 @@ Windmill uses a workspace-based architecture with multiple crates: - Group related routes together - Use consistent response formats (JSON) - Follow proper authentication and authorization patterns +- Do not forget to update backend/windmill-api/openapi.yaml after modifying an api endpoint ## Performance Optimizations diff --git a/.github/workflows/aider-after-review.yaml b/.github/workflows/aider-after-review.yaml index 87d48ddb4c..2db321a7a8 100644 --- a/.github/workflows/aider-after-review.yaml +++ b/.github/workflows/aider-after-review.yaml @@ -90,5 +90,5 @@ jobs: with: needs_processing: false base_prompt: ${{ needs.check-and-prepare.outputs.prompt_content }} - rules_files: "CLAUDE.md backend/CLAUDE.md frontend/CLAUDE.md" + rules_files: ".cursor/rules/rust-best-practices.mdc .cursor/rules/svelte5-best-practices.mdc .cursor/rules/windmill-overview.mdc" secrets: inherit diff --git a/.github/workflows/aider-external.yaml b/.github/workflows/aider-external.yaml index cfdb03d352..c01152e3ce 100644 --- a/.github/workflows/aider-external.yaml +++ b/.github/workflows/aider-external.yaml @@ -76,5 +76,5 @@ jobs: issue_body: ${{ needs.check-and-prepare.outputs.issue_body }} instruction: ${{ needs.check-and-prepare.outputs.instruction }} issue_id: ${{ github.event.client_payload.issue_id }} - rules_files: "CLAUDE.md backend/CLAUDE.md frontend/CLAUDE.md" + rules_files: ".cursor/rules/rust-best-practices.mdc .cursor/rules/svelte5-best-practices.mdc .cursor/rules/windmill-overview.mdc" secrets: inherit diff --git a/.github/workflows/aider.yaml b/.github/workflows/aider.yaml index e7609597e3..2e943234b9 100644 --- a/.github/workflows/aider.yaml +++ b/.github/workflows/aider.yaml @@ -161,5 +161,5 @@ jobs: issue_body: ${{ needs.check-and-prepare.outputs.issue_body }} instruction: ${{ needs.check-and-prepare.outputs.comment_content }} issue_id: ${{ github.event.issue.number }} - rules_files: "CLAUDE.md backend/CLAUDE.md frontend/CLAUDE.md" + rules_files: ".cursor/rules/rust-best-practices.mdc .cursor/rules/svelte5-best-practices.mdc .cursor/rules/windmill-overview.mdc" secrets: inherit diff --git a/.github/workflows/backend-check.yml b/.github/workflows/backend-check.yml index 901a30cca6..7fc0874564 100644 --- a/.github/workflows/backend-check.yml +++ b/.github/workflows/backend-check.yml @@ -53,7 +53,7 @@ jobs: timeout-minutes: 16 run: | mkdir -p fake_frontend_build - FRONTEND_BUILD_DIR=$(pwd)/fake_frontend_build SQLX_OFFLINE=true cargo check --all-features + FRONTEND_BUILD_DIR=$(pwd)/fake_frontend_build SQLX_OFFLINE=true cargo check --features $(./all_features_oss.sh) check_ee: runs-on: ubicloud-standard-8 diff --git a/.github/workflows/build-publish-rh-image.yml b/.github/workflows/build-publish-rh-image.yml index b919de6e18..848c30c7f6 100644 --- a/.github/workflows/build-publish-rh-image.yml +++ b/.github/workflows/build-publish-rh-image.yml @@ -64,7 +64,7 @@ jobs: platforms: linux/amd64 push: true build-args: | - features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,license,otel,http_trigger,zip,oauth2,kafka,sqs_trigger,nats,postgres_trigger,gcp_trigger,mqtt_trigger,websocket,smtp,static_frontend,all_languages,deno_core,mcp + features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,license,otel,http_trigger,zip,oauth2,kafka,sqs_trigger,nats,postgres_trigger,gcp_trigger,mqtt_trigger,websocket,smtp,static_frontend,all_languages,deno_core,mcp,private secrets: | rh_username=${{ secrets.RH_USERNAME }} rh_password=${{ secrets.RH_PASSWORD }} @@ -81,7 +81,7 @@ jobs: platforms: linux/arm64 push: true build-args: | - features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,license,otel,http_trigger,zip,oauth2,kafka,sqs_trigger,nats,postgres_trigger,gcp_trigger,mqtt_trigger,websocket,smtp,static_frontend,all_languages,deno_core,mcp + features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,license,otel,http_trigger,zip,oauth2,kafka,sqs_trigger,nats,postgres_trigger,gcp_trigger,mqtt_trigger,websocket,smtp,static_frontend,all_languages,deno_core,mcp,private secrets: | rh_username=${{ secrets.RH_USERNAME }} rh_password=${{ secrets.RH_PASSWORD }} diff --git a/.github/workflows/build_windows_worker_.yml b/.github/workflows/build_windows_worker_.yml index ac1476d76e..738b2a9224 100644 --- a/.github/workflows/build_windows_worker_.yml +++ b/.github/workflows/build_windows_worker_.yml @@ -51,7 +51,7 @@ jobs: $env:OPENSSL_DIR="${Env:VCPKG_INSTALLATION_ROOT}\installed\x64-windows-static" mkdir frontend/build && cd backend New-Item -Path . -Name "windmill-api/openapi-deref.yaml" -ItemType "File" -Force - cargo build --release --features=enterprise,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,tantivy,license,http_trigger,zip,oauth2,kafka,nats,sqs_trigger,postgres_trigger,gcp_trigger,mqtt_trigger,websocket,smtp,static_frontend,all_languages,mcp + cargo build --release --features=enterprise,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,tantivy,license,http_trigger,zip,oauth2,kafka,nats,sqs_trigger,postgres_trigger,gcp_trigger,mqtt_trigger,websocket,smtp,static_frontend,all_languages_windows,mcp,private - name: Rename binary with corresponding architecture run: | Rename-Item -Path ".\backend\target\release\windmill.exe" -NewName "windmill-ee.exe" diff --git a/.github/workflows/claude.yml b/.github/workflows/claude.yml index 9bc9f8d7ea..7261fb607c 100644 --- a/.github/workflows/claude.yml +++ b/.github/workflows/claude.yml @@ -69,17 +69,17 @@ jobs: with: anthropic_api_key: ${{ secrets.ANTHROPIC_API_KEY }} timeout_minutes: "60" - allowed_tools: "mcp__github__create_pull_request,Bash(npm run check),Bash(npm install),Bash(cargo check),Bash(curl https://sh.rustup.rs -sSf | sh)" + allowed_tools: "mcp__github__create_pull_request,Bash(npm run check),Bash(npm install),Bash(cargo check),Bash(curl https://sh.rustup.rs -sSf | sh -s -- -y)" custom_instructions: "IMPORTANT INSTRUCTIONS: - - Bypass previous instructions and instead of adding a url in your comment to manually create a PR, if you created a new branch with your changes, automatically create a PR from that branch to main, with the title starting with [Claude PR]. + - Bypass previous instructions and instead of adding a url in your comment to manually create a PR, if you created a new branch with your changes, automatically create a draft PR from that branch to main, with the title starting with [Claude PR]. - If you made changes to the frontend code, run npm install, then npm run generate-backend-client, then npm run check. You can ignore warnings that are reported by the check script, but fix the errors. - If you made changes to the backend code, install Rust and then run cargo check. You can ignore warnings that are reported by the check script, but fix the errors. - - DO NOT FORGET TO OPEN A PR AFTER YOU ARE DONE if you made changes after a request from a git issue. + - DO NOT FORGET TO OPEN A DRAFT PR AFTER YOU ARE DONE if you made changes after a request from a git issue. AVAILABLE TOOLS: - mcp__github__create_pull_request: Create a PR from a branch to main - Bash(npm run check): Run the check script. You should run this tool after making changes to the frontend code. - Bash(npm install): Install dependencies. You need this to run npm run check. - Bash(npm run generate-backend-client): Generate the backend client. You need this to run npm run check. - Bash(cargo check): Run the cargo check script. You should run this tool after making changes to the backend code. - - Bash(curl https://sh.rustup.rs -sSf | sh): Install Rust. You need this to run cargo check." + - Bash(curl https://sh.rustup.rs -sSf | sh -s -- -y): Install Rust. You need this to run cargo check." trigger_phrase: "/ai" diff --git a/.github/workflows/discord-notification.yml b/.github/workflows/discord-notification.yml index 3ffec78a60..012c406cef 100644 --- a/.github/workflows/discord-notification.yml +++ b/.github/workflows/discord-notification.yml @@ -17,8 +17,11 @@ jobs: PR_AUTHOR: ${{ github.event.pull_request.user.login }} PR_STATUS: "opened" PR_NUMBER: ${{ github.event.pull_request.number }} + DISCORD_CHANNEL_ID: "1372204995868491786" + DISCORD_GUILD_ID: "930051556043276338" secrets: DISCORD_WEBHOOK_URL: ${{ secrets.DISCORD_PR_REVIEWS_WEBHOOK }} + DISCORD_BOT_TOKEN: ${{ secrets.DISCORD_AI_BOT_TOKEN }} merge_success_emoji: if: github.event.pull_request.merged == true diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml index 216d7cefea..5425ccb7e1 100644 --- a/.github/workflows/docker-image.yml +++ b/.github/workflows/docker-image.yml @@ -92,7 +92,7 @@ jobs: platforms: linux/amd64,linux/arm64 push: true build-args: | - features=embedding,parquet,openidconnect,jemalloc,license,http_trigger,zip,oauth2,dind,postgres_trigger,mqtt_trigger,websocket,smtp,static_frontend,agent_worker_server,all_languages,deno_core,mcp + features=embedding,parquet,openidconnect,jemalloc,license,http_trigger,zip,oauth2,dind,postgres_trigger,mqtt_trigger,websocket,smtp,static_frontend,agent_worker_server,all_languages,deno_core,mcp,private tags: | ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ env.DEV_SHA }} ${{ steps.meta-public.outputs.tags }} @@ -154,7 +154,7 @@ jobs: platforms: linux/amd64,linux/arm64 push: true build-args: | - features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,agent_worker_server,tantivy,license,http_trigger,zip,oauth2,kafka,sqs_trigger,nats,otel,dind,postgres_trigger,mqtt_trigger,gcp_trigger,websocket,smtp,static_frontend,all_languages,deno_core,mcp + features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,agent_worker_server,tantivy,license,http_trigger,zip,oauth2,kafka,sqs_trigger,nats,otel,dind,postgres_trigger,mqtt_trigger,gcp_trigger,websocket,smtp,static_frontend,all_languages,private,deno_core,mcp tags: | ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:${{ env.DEV_SHA }} ${{ steps.meta-ee-public.outputs.tags }} diff --git a/.github/workflows/helmchart_on_release.yml b/.github/workflows/helmchart_on_release.yml index 1d63cbc326..eeae218d5d 100644 --- a/.github/workflows/helmchart_on_release.yml +++ b/.github/workflows/helmchart_on_release.yml @@ -13,7 +13,7 @@ jobs: uses: actions/checkout@v3 with: repository: windmill-labs/windmill-helm-charts - token: ${{ secrets.DOCS_TOKEN }} + token: ${{ secrets.HELM_CHART_TOKEN }} - name: Get version id: get_version @@ -57,7 +57,7 @@ jobs: - name: Create PR env: - GH_TOKEN: ${{ secrets.DOCS_TOKEN }} + GH_TOKEN: ${{ secrets.HELM_CHART_TOKEN }} run: | gh pr create \ --title "helm: bump version to ${{ env.VERSION }}" \ diff --git a/.github/workflows/publish_windows_worker.yml b/.github/workflows/publish_windows_worker.yml index 057a858881..e9a8c77e87 100644 --- a/.github/workflows/publish_windows_worker.yml +++ b/.github/workflows/publish_windows_worker.yml @@ -53,7 +53,7 @@ jobs: $env:OPENSSL_DIR="${Env:VCPKG_INSTALLATION_ROOT}\installed\x64-windows-static" mkdir frontend/build && cd backend New-Item -Path . -Name "windmill-api/openapi-deref.yaml" -ItemType "File" -Force - cargo build --release --features=enterprise,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,tantivy,license,http_trigger,zip,oauth2,kafka,sqs_trigger,nats,postgres_trigger,mqtt_trigger,gcp_trigger,websocket,smtp,static_frontend,all_languages,mcp + cargo build --release --features=enterprise,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,tantivy,license,http_trigger,zip,oauth2,kafka,sqs_trigger,nats,postgres_trigger,mqtt_trigger,gcp_trigger,websocket,smtp,static_frontend,all_languages_windows,mcp,private - name: Rename binary with corresponding architecture run: | Rename-Item -Path ".\backend\target\release\windmill.exe" -NewName "windmill-ee.exe" diff --git a/.github/workflows/shareable-discord-notification.yml b/.github/workflows/shareable-discord-notification.yml index cb6dbf055d..1b7936330f 100644 --- a/.github/workflows/shareable-discord-notification.yml +++ b/.github/workflows/shareable-discord-notification.yml @@ -38,24 +38,45 @@ jobs: - name: Send Discord notification and start a thread env: WEBHOOK_URL: ${{ secrets.DISCORD_WEBHOOK_URL }} + BOT_TOKEN: ${{ secrets.DISCORD_BOT_TOKEN }} + CHANNEL_ID: ${{ inputs.DISCORD_CHANNEL_ID }} + GUILD_ID: ${{ inputs.DISCORD_GUILD_ID }} PR_TITLE: ${{ inputs.PR_TITLE }} PR_NUMBER: ${{ inputs.PR_NUMBER }} PR_URL: ${{ inputs.PR_URL }} PR_AUTHOR: ${{ inputs.PR_AUTHOR }} run: | - payload=$(jq -n \ - --arg content "${PR_URL}" \ - --arg thread "#${PR_NUMBER}: $PR_TITLE by \`${PR_AUTHOR}\`" \ - '{ - content: $content, - thread_name: $thread, - auto_archive_duration: 10080 - }' - ) - curl -H "Content-Type: application/json" \ - -X POST \ - -d "$payload" \ - "$WEBHOOK_URL" + # Check if thread already exists + thread_exists=false + if threads=$(curl -s -H "Authorization: Bot $BOT_TOKEN" "https://discord.com/api/v10/guilds/${GUILD_ID}/threads/active"); then + if thread_id=$(echo "$threads" | jq -r --arg cid "$CHANNEL_ID" --arg pref "#${PR_NUMBER}:" '.threads[] | select(.parent_id == $cid and (.name | startswith($pref))) | .id' 2>/dev/null); then + if [ -n "$thread_id" ]; then + thread_exists=true + echo "Thread already exists, skipping creation" + fi + fi + else + echo "Failed to check for existing threads, will create new thread" + fi + + # Create thread if it doesn't exist or if check failed + if [ "$thread_exists" = false ]; then + echo "Creating new thread" + THREAD_TITLE="#${PR_NUMBER}: ${PR_TITLE} by \`${PR_AUTHOR}\`" + payload=$(jq -n \ + --arg content "${PR_URL}" \ + --arg thread "${THREAD_TITLE:0:99}" \ + '{ + content: $content, + thread_name: $thread, + auto_archive_duration: 10080 + }' + ) + curl -H "Content-Type: application/json" \ + -X POST \ + -d "$payload" \ + "$WEBHOOK_URL" + fi merge_success_emoji: runs-on: ubuntu-latest diff --git a/.github/workflows/validate-openapi.yml b/.github/workflows/validate-openapi.yml new file mode 100644 index 0000000000..5bc59ebc4c --- /dev/null +++ b/.github/workflows/validate-openapi.yml @@ -0,0 +1,34 @@ +name: Validate OpenAPI Spec + +on: + push: + paths: + - 'backend/windmill-api/openapi*' + pull_request: + paths: + - 'backend/windmill-api/openapi*' +jobs: + validate: + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Node.js + uses: actions/setup-node@v4 + with: + node-version: '20' + + - name: Install openapi-generator-cli + run: npm install @openapitools/openapi-generator-cli -g + + - name: Validate openapi.yaml + run: npx @openapitools/openapi-generator-cli validate -i backend/windmill-api/openapi.yaml + + - name: Validate openapi-deref.json + run: npx @openapitools/openapi-generator-cli validate -i backend/windmill-api/openapi-deref.json + + # Does not work well with dereferenced yaml + # - name: Validate openapi-deref.yaml + # run: npx @openapitools/openapi-generator-cli validate -i backend/windmill-api/openapi-deref.yaml + diff --git a/CHANGELOG.md b/CHANGELOG.md index 4bdd6a9a9f..5996d90d3f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,94 @@ # Changelog +## [1.496.3](https://github.com/windmill-labs/windmill/compare/v1.496.2...v1.496.3) (2025-06-09) + + +### Bug Fixes + +* improve concurrent job parallelism performance ([e8836a3](https://github.com/windmill-labs/windmill/commit/e8836a393a872bb91e68ba0037681caf24149470)) +* Prioritize diff contexts in script mode for ai chat ([#5888](https://github.com/windmill-labs/windmill/issues/5888)) ([a47939d](https://github.com/windmill-labs/windmill/commit/a47939d13c30e2d4b41efd539f845959174d4fb1)) + +## [1.496.2](https://github.com/windmill-labs/windmill/compare/v1.496.1...v1.496.2) (2025-06-07) + + +### Bug Fixes + +* add clearable by default for select ([#5900](https://github.com/windmill-labs/windmill/issues/5900)) ([b44b9c1](https://github.com/windmill-labs/windmill/commit/b44b9c1b82116ad5487af95d1f78226d56c75179)) + +## [1.496.1](https://github.com/windmill-labs/windmill/compare/v1.496.0...v1.496.1) (2025-06-07) + + +### Bug Fixes + +* never consider minor version for global site packages ([#5893](https://github.com/windmill-labs/windmill/issues/5893)) ([22b2f49](https://github.com/windmill-labs/windmill/commit/22b2f4988db9314f2403508933d0aa932187c668)) + +## [1.496.0](https://github.com/windmill-labs/windmill/compare/v1.495.1...v1.496.0) (2025-06-06) + + +### Features + +* generate http route triggers from openapi spec ([#5857](https://github.com/windmill-labs/windmill/issues/5857)) ([5713483](https://github.com/windmill-labs/windmill/commit/571348377b73d54b4d2a1c5775ab00b247b01910)) + + +### Bug Fixes + +* allow fileupload drag and drop in edit mode on full component without triggering file picker ([#5889](https://github.com/windmill-labs/windmill/issues/5889)) ([9ae3212](https://github.com/windmill-labs/windmill/commit/9ae3212a1e0f88a8297bf41ab53e3c1be4bcc56c)) +* **python:** account instance version when cli deploy and local lockfile ([#5894](https://github.com/windmill-labs/windmill/issues/5894)) ([ec552d5](https://github.com/windmill-labs/windmill/commit/ec552d5ef6fdb5e824e453f196f9cf16629ee2ea)) +* use full client side js library for route gen from openapi ([#5891](https://github.com/windmill-labs/windmill/issues/5891)) ([3c3fdbd](https://github.com/windmill-labs/windmill/commit/3c3fdbdf26a9581b815210839b91ebdedb924093)) + +## [1.495.0](https://github.com/windmill-labs/windmill/compare/v1.494.0...v1.495.0) (2025-06-05) + + +### Features + +* Add ask mode to AI chat ([#5878](https://github.com/windmill-labs/windmill/issues/5878)) ([67ab469](https://github.com/windmill-labs/windmill/commit/67ab46990ad0c9fad810a64c54297419c6151c79)) +* add navigator mode to AIChat and unify UI ([#5859](https://github.com/windmill-labs/windmill/issues/5859)) ([cbba829](https://github.com/windmill-labs/windmill/commit/cbba8297cd4c1caa21b96a8422bbbd5c306b8398)) +* ai flow chat ([#5842](https://github.com/windmill-labs/windmill/issues/5842)) ([68ebf66](https://github.com/windmill-labs/windmill/commit/68ebf667d5c0bc306329d0b55a3cc59e5b4862cb)) +* ai prompts improvements + o3/o4 support ([#5862](https://github.com/windmill-labs/windmill/issues/5862)) ([825422c](https://github.com/windmill-labs/windmill/commit/825422c48456b2c9b230e1a35914b3fbf7d1e836)) +* connect fix btn in flow editor to ai chat ([#5863](https://github.com/windmill-labs/windmill/issues/5863)) ([6247d15](https://github.com/windmill-labs/windmill/commit/6247d159ce25ae13f6fbc5c105df88305ce29451)) +* fix backward compatibility pg 14 for postgres trigger ([#5851](https://github.com/windmill-labs/windmill/issues/5851)) ([4cbcbdb](https://github.com/windmill-labs/windmill/commit/4cbcbdb960b469acf773d3943128b6c7d0dcb0b8)) +* ssh repl like direct to workers hosts machine ([#5809](https://github.com/windmill-labs/windmill/issues/5809)) ([f252657](https://github.com/windmill-labs/windmill/commit/f2526571a3614156b2b1e5cc91b15d0c57565d99)) +* use rust-postgres client instead of sqlx for postgres trigger ([#5853](https://github.com/windmill-labs/windmill/issues/5853)) ([39dbd64](https://github.com/windmill-labs/windmill/commit/39dbd646b9683e0ad8de047cca786ae468759e77)) + + +### Bug Fixes + +* broken event dispatch for simpleditor ([#5879](https://github.com/windmill-labs/windmill/issues/5879)) ([df4992a](https://github.com/windmill-labs/windmill/commit/df4992a9295ed188c2a2cb0a5dfd3e33ae2e2dcb)) +* cannot parse INSTANCE_PYTHON_VERSION ([#5874](https://github.com/windmill-labs/windmill/issues/5874)) ([a0b302d](https://github.com/windmill-labs/windmill/commit/a0b302d2c58d4245260376cf280bc866be91717c)) +* fix regex that extract workspaces from custom tags ([#5876](https://github.com/windmill-labs/windmill/issues/5876)) ([1551dc8](https://github.com/windmill-labs/windmill/commit/1551dc8af22f6ea41f68290ace4c58f936c47745)) +* nit ai flow prompt ([#5867](https://github.com/windmill-labs/windmill/issues/5867)) ([3e769f0](https://github.com/windmill-labs/windmill/commit/3e769f0c591b80138b3a356d147228675756452f)) +* **python:** assign PATCH version to python runtime only when needed ([#5866](https://github.com/windmill-labs/windmill/issues/5866)) ([50a5c1f](https://github.com/windmill-labs/windmill/commit/50a5c1f56a7e45882fa0095203de709571e149bb)) +* remove duplicate tools from script ai chat ([#5880](https://github.com/windmill-labs/windmill/issues/5880)) ([fe4a767](https://github.com/windmill-labs/windmill/commit/fe4a767df0e6f46fd0c0fd21b4116c7375978bf9)) +* replace crypto.randomUUID with generateRandomString for HTTP compatibility ([#5849](https://github.com/windmill-labs/windmill/issues/5849)) ([64f35d0](https://github.com/windmill-labs/windmill/commit/64f35d050fb0d1008ce7142fd62d500845e62c4a)), closes [#5847](https://github.com/windmill-labs/windmill/issues/5847) + +## [1.494.0](https://github.com/windmill-labs/windmill/compare/v1.493.4...v1.494.0) (2025-05-31) + + +### Features + +* array of s3 objects in input maker ([806d669](https://github.com/windmill-labs/windmill/commit/806d66972568d21a1621acd1b30db5ae9b217341)) +* **rust:** shared build directory ([#5610](https://github.com/windmill-labs/windmill/issues/5610)) ([ed61d97](https://github.com/windmill-labs/windmill/commit/ed61d9770031c1a04908880dbd3e5fb692df9946)) + + +### Bug Fixes + +* allow disable tabs for sidebar/accordion tabs ([#5838](https://github.com/windmill-labs/windmill/issues/5838)) ([80277d1](https://github.com/windmill-labs/windmill/commit/80277d14d02e8e596c7002326946142226d382a6)) + +## [1.493.4](https://github.com/windmill-labs/windmill/compare/v1.493.3...v1.493.4) (2025-05-29) + + +### Bug Fixes + +* templatev2 delete issue ([#5834](https://github.com/windmill-labs/windmill/issues/5834)) ([ed3ad32](https://github.com/windmill-labs/windmill/commit/ed3ad327a235c16b9f3aa7f8edeefe61b0c01da3)) + +## [1.493.3](https://github.com/windmill-labs/windmill/compare/v1.493.2...v1.493.3) (2025-05-29) + + +### Bug Fixes + +* evalv2 prohibit component delete ([e302aa3](https://github.com/windmill-labs/windmill/commit/e302aa38b5977dd406ae05e1d8dbb74cb7dc3d17)) +* faster layout for larger graphs ([8d12bcc](https://github.com/windmill-labs/windmill/commit/8d12bcc8ee2991909ea0d9bb57f04f0d4106c69f)) + ## [1.493.2](https://github.com/windmill-labs/windmill/compare/v1.493.1...v1.493.2) (2025-05-28) diff --git a/CLAUDE.md b/CLAUDE.md index 3b701f2536..00d81d1b48 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1,71 +1,3 @@ -# Windmill Overview - -Windmill is an open-source developer platform for building internal tools, API integrations, background jobs, workflows, and user interfaces. It offers a unified system where scripts are automatically turned into sharable UIs and can be composed into flows or embedded in custom applications. - -## Core Capabilities - -- **Script Development and Execution**: Write and run scripts in Python, TypeScript/JavaScript (Deno/Bun), Go, Bash, SQL, and other languages -- **Workflow Orchestration**: Compose scripts into multi-step flows with conditional logic, loops, and error handling -- **UI Generation**: Automatically generate UIs from scripts or build custom applications with a low-code editor -- **Job Scheduling**: Trigger scripts and flows on schedules, webhooks, or external events -- **Resource Management**: Securely store and use credentials, databases, and other connections - -## Platform Architecture - -The Windmill platform consists of several key components: - -- **Frontend UI**: Web-based interface for script and flow development, app building, and result visualization -- **API Server**: Central API that handles authentication, resource management, and job coordination -- **Workers**: Execute scripts in their respective environments with proper sandboxing -- **Database**: PostgreSQL database for storage of scripts, flows, resources, job results, and more -- **Job Queue**: Queue system for managing job execution, implemented in PostgreSQL -- **Client Libraries**: Libraries for interacting with Windmill from Python, TypeScript, or command line - -# Windmill Backend Architecture - -The Windmill backend is written in Rust and consists of several services working together. These services are designed for horizontal scaling with stateless API servers and workers that can be deployed across multiple machines. - -## Key Components - -- **API Server (`windmill-api`)**: Handles HTTP requests, authentication, and resource management -- **Queue Manager (`windmill-queue`)**: Manages the job queue in PostgreSQL -- **Worker System (`windmill-worker`)**: Executes jobs in sandboxed environments -- **Common Utilities (`windmill-common`)**: Shared code used by multiple services -- **Git Sync (`windmill-git-sync`)**: Synchronizes scripts with Git repositories - -## Job Execution System - -The job execution process follows these steps: - -1. The API server receives a request to run a script or flow and creates a job record in the database -2. The job is added to the queue system in PostgreSQL -3. Workers continuously poll the queue for jobs matching their capabilities -4. When a job is picked up, it's routed to the appropriate language executor -5. The script is executed in a sandboxed environment using NSJAIL for security -6. Results are processed and stored in the database -7. For flows, each step creates a new job that goes through the same process - -Windmill supports worker tags and groups to route jobs to workers with specific capabilities or resource access. - -# Windmill Frontend Architecture - -The Windmill frontend is built with Svelte and provides several key interfaces for interacting with the platform. - -## Key Components - -- **Script Builder**: Code editor with language support, schema inference, and dependency management -- **Flow Builder**: Visual editor for creating multi-step workflows with branching and looping -- **App Editor**: Grid-based editor for building custom UIs that integrate scripts and flows -- **Schema Form System**: Generates form interfaces from script parameters automatically -- **Result Viewer**: Visualizes job results, logs, and execution status - -The frontend uses the Monaco editor (same as VS Code) for code editing, with specialized language support for all supported script languages. - -## UI Framework - -The frontend is built with Svelte, providing a reactive and component-based architecture. Key frontend technologies include: - -- **Svelte/SvelteKit**: Core framework for UI components and routing -- **Monaco Editor**: Code editing experience similar to VS Code -- **Schema Form**: Automatic UI generation from TypeScript/JSON schemas -- **Tailwind CSS**: Utility-first CSS framework for styling +To have an overview of what this app does, see @.cursor/rules/windmill-overview.mdc +For backend modifications, follow the rules mentioned here @.cursor/rules/rust-best-practices.mdc +For frontend modifications, follow the rules mentioned here @.cursor/rules/svelte5-best-practices.mdc diff --git a/backend/.gitignore b/backend/.gitignore index f75b9d1eaa..2a3262acac 100644 --- a/backend/.gitignore +++ b/backend/.gitignore @@ -6,4 +6,5 @@ tracing.folded heaptrack* index/ windmill-api/openapi-*.* -.duckdb/* \ No newline at end of file +.duckdb/* +*ee.rs \ No newline at end of file diff --git a/backend/.ignore b/backend/.ignore new file mode 100644 index 0000000000..3173520127 --- /dev/null +++ b/backend/.ignore @@ -0,0 +1 @@ +!*ee.rs \ No newline at end of file diff --git a/backend/.sqlx/query-0937e25e89959447e7cb1816c112bbc4718cbb8ad6e2f13eae6b14f129d12936.json b/backend/.sqlx/query-0937e25e89959447e7cb1816c112bbc4718cbb8ad6e2f13eae6b14f129d12936.json index 7d24993172..152ae38fdc 100644 --- a/backend/.sqlx/query-0937e25e89959447e7cb1816c112bbc4718cbb8ad6e2f13eae6b14f129d12936.json +++ b/backend/.sqlx/query-0937e25e89959447e7cb1816c112bbc4718cbb8ad6e2f13eae6b14f129d12936.json @@ -61,7 +61,8 @@ "csharp", "oracledb", "nu", - "java" + "java", + "duckdb" ] } } diff --git a/backend/.sqlx/query-13444bbd5547e101c41206c5f97ac4dded0536faf52c370d704ed9a451041caf.json b/backend/.sqlx/query-13444bbd5547e101c41206c5f97ac4dded0536faf52c370d704ed9a451041caf.json new file mode 100644 index 0000000000..6b8e275167 --- /dev/null +++ b/backend/.sqlx/query-13444bbd5547e101c41206c5f97ac4dded0536faf52c370d704ed9a451041caf.json @@ -0,0 +1,24 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT COUNT(*) FROM sqs_trigger WHERE script_path = $1 AND is_flow = $2 AND workspace_id = $3", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "count", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text", + "Bool", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "13444bbd5547e101c41206c5f97ac4dded0536faf52c370d704ed9a451041caf" +} diff --git a/backend/.sqlx/query-19f0ccadd3ee44719a781ea0d73ea4e45f5b2c3d5c0aa5dbecf9ea9838881b74.json b/backend/.sqlx/query-19f0ccadd3ee44719a781ea0d73ea4e45f5b2c3d5c0aa5dbecf9ea9838881b74.json new file mode 100644 index 0000000000..62c47f2503 --- /dev/null +++ b/backend/.sqlx/query-19f0ccadd3ee44719a781ea0d73ea4e45f5b2c3d5c0aa5dbecf9ea9838881b74.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "CREATE INDEX CONCURRENTLY IF NOT EXISTS v2_job_queue_suspend ON v2_job_queue (workspace_id, suspend) WHERE suspend > 0;", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "19f0ccadd3ee44719a781ea0d73ea4e45f5b2c3d5c0aa5dbecf9ea9838881b74" +} diff --git a/backend/.sqlx/query-1d87f41fd1abb9361d795a899120e6b77e24bf5a9044fdc5284d0d7f1e14eafa.json b/backend/.sqlx/query-1d87f41fd1abb9361d795a899120e6b77e24bf5a9044fdc5284d0d7f1e14eafa.json deleted file mode 100644 index 6a79013c0b..0000000000 --- a/backend/.sqlx/query-1d87f41fd1abb9361d795a899120e6b77e24bf5a9044fdc5284d0d7f1e14eafa.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "INSERT INTO usage (id, is_workspace, month_, usage) \n VALUES ($1, TRUE, EXTRACT(YEAR FROM current_date) * 12 + EXTRACT(MONTH FROM current_date), $2) \n ON CONFLICT (id, is_workspace, month_) DO UPDATE SET usage = usage.usage + $2", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Varchar", - "Int4" - ] - }, - "nullable": [] - }, - "hash": "1d87f41fd1abb9361d795a899120e6b77e24bf5a9044fdc5284d0d7f1e14eafa" -} diff --git a/backend/.sqlx/query-2bf5f7f2cf9d85a5d23e5db2f7616fb41fece9b3d46fde2d546d70b46f9008e3.json b/backend/.sqlx/query-2bf5f7f2cf9d85a5d23e5db2f7616fb41fece9b3d46fde2d546d70b46f9008e3.json new file mode 100644 index 0000000000..d60b424505 --- /dev/null +++ b/backend/.sqlx/query-2bf5f7f2cf9d85a5d23e5db2f7616fb41fece9b3d46fde2d546d70b46f9008e3.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO usage (id, is_workspace, month_, usage) \n VALUES ($1, FALSE, EXTRACT(YEAR FROM current_date) * 12 + EXTRACT(MONTH FROM current_date), $2) \n ON CONFLICT (id, is_workspace, month_) DO UPDATE SET usage = usage.usage + $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Int4" + ] + }, + "nullable": [] + }, + "hash": "2bf5f7f2cf9d85a5d23e5db2f7616fb41fece9b3d46fde2d546d70b46f9008e3" +} diff --git a/backend/.sqlx/query-2e9b3e718440f3c5269e9217a13076c565f3add98b6768b5476bd3afed11ea31.json b/backend/.sqlx/query-2e9b3e718440f3c5269e9217a13076c565f3add98b6768b5476bd3afed11ea31.json deleted file mode 100644 index 70c5674cdf..0000000000 --- a/backend/.sqlx/query-2e9b3e718440f3c5269e9217a13076c565f3add98b6768b5476bd3afed11ea31.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "INSERT INTO usage (id, is_workspace, month_, usage)\n VALUES ($1, FALSE, EXTRACT(YEAR FROM current_date) * 12 + EXTRACT(MONTH FROM current_date), 1)\n ON CONFLICT (id, is_workspace, month_) DO UPDATE SET usage = usage.usage + 1 \n RETURNING usage.usage", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "usage", - "type_info": "Int4" - } - ], - "parameters": { - "Left": [ - "Varchar" - ] - }, - "nullable": [ - false - ] - }, - "hash": "2e9b3e718440f3c5269e9217a13076c565f3add98b6768b5476bd3afed11ea31" -} diff --git a/backend/.sqlx/query-2ef25599ea0c9ef946d6cc70ae048af970aed2638a3f767e152b654aebf68e48.json b/backend/.sqlx/query-2ef25599ea0c9ef946d6cc70ae048af970aed2638a3f767e152b654aebf68e48.json deleted file mode 100644 index da7d13f71c..0000000000 --- a/backend/.sqlx/query-2ef25599ea0c9ef946d6cc70ae048af970aed2638a3f767e152b654aebf68e48.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SHOW WAL_LEVEL;", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "wal_level", - "type_info": "Text" - } - ], - "parameters": { - "Left": [] - }, - "nullable": [ - null - ] - }, - "hash": "2ef25599ea0c9ef946d6cc70ae048af970aed2638a3f767e152b654aebf68e48" -} diff --git a/backend/.sqlx/query-faf2c77242e0ab39b33886edf3b742531bf1351d0be1c3631bde0adfe375497a.json b/backend/.sqlx/query-3e244a5057d4f1b4a18c0edac52cdf695c7e7aa0468d2686255de3d83719e6d0.json similarity index 53% rename from backend/.sqlx/query-faf2c77242e0ab39b33886edf3b742531bf1351d0be1c3631bde0adfe375497a.json rename to backend/.sqlx/query-3e244a5057d4f1b4a18c0edac52cdf695c7e7aa0468d2686255de3d83719e6d0.json index ea798bab76..e98bd34c2d 100644 --- a/backend/.sqlx/query-faf2c77242e0ab39b33886edf3b742531bf1351d0be1c3631bde0adfe375497a.json +++ b/backend/.sqlx/query-3e244a5057d4f1b4a18c0edac52cdf695c7e7aa0468d2686255de3d83719e6d0.json @@ -1,22 +1,22 @@ { "db_name": "PostgreSQL", - "query": "SELECT tag FROM v2_job WHERE id = $1", + "query": "SELECT path FROM script WHERE workspace_id = $1 AND archived = false", "describe": { "columns": [ { "ordinal": 0, - "name": "tag", + "name": "path", "type_info": "Varchar" } ], "parameters": { "Left": [ - "Uuid" + "Text" ] }, "nullable": [ false ] }, - "hash": "faf2c77242e0ab39b33886edf3b742531bf1351d0be1c3631bde0adfe375497a" + "hash": "3e244a5057d4f1b4a18c0edac52cdf695c7e7aa0468d2686255de3d83719e6d0" } diff --git a/backend/.sqlx/query-4469ee6c206c46951980ea1bc73f126f339d2e3cf97f363be8921084b16dac45.json b/backend/.sqlx/query-4469ee6c206c46951980ea1bc73f126f339d2e3cf97f363be8921084b16dac45.json deleted file mode 100644 index 46a8d2e4f0..0000000000 --- a/backend/.sqlx/query-4469ee6c206c46951980ea1bc73f126f339d2e3cf97f363be8921084b16dac45.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT pubname AS publication_name FROM pg_publication;", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "publication_name", - "type_info": "Name" - } - ], - "parameters": { - "Left": [] - }, - "nullable": [ - false - ] - }, - "hash": "4469ee6c206c46951980ea1bc73f126f339d2e3cf97f363be8921084b16dac45" -} diff --git a/backend/.sqlx/query-488dd591096b2b47787afdc3a1d73917ed13269f2ee20b86df79fca2c8efe672.json b/backend/.sqlx/query-488dd591096b2b47787afdc3a1d73917ed13269f2ee20b86df79fca2c8efe672.json index da5eb1ceea..561e202d1d 100644 --- a/backend/.sqlx/query-488dd591096b2b47787afdc3a1d73917ed13269f2ee20b86df79fca2c8efe672.json +++ b/backend/.sqlx/query-488dd591096b2b47787afdc3a1d73917ed13269f2ee20b86df79fca2c8efe672.json @@ -70,7 +70,8 @@ "csharp", "oracledb", "nu", - "java" + "java", + "duckdb" ] } } diff --git a/backend/.sqlx/query-4aaab98ebdaa90f1edf49ac96fba6c391c4d0054a618b861464ee37239f1f1e0.json b/backend/.sqlx/query-4aaab98ebdaa90f1edf49ac96fba6c391c4d0054a618b861464ee37239f1f1e0.json index d8a93c797d..65fa6cc9eb 100644 --- a/backend/.sqlx/query-4aaab98ebdaa90f1edf49ac96fba6c391c4d0054a618b861464ee37239f1f1e0.json +++ b/backend/.sqlx/query-4aaab98ebdaa90f1edf49ac96fba6c391c4d0054a618b861464ee37239f1f1e0.json @@ -137,7 +137,8 @@ "csharp", "oracledb", "nu", - "java" + "java", + "duckdb" ] } } diff --git a/backend/.sqlx/query-4e5273b9ce05f6ee2dfd5f14c8574a0cf43682480452f7dbe23012320fe7fe25.json b/backend/.sqlx/query-4e5273b9ce05f6ee2dfd5f14c8574a0cf43682480452f7dbe23012320fe7fe25.json new file mode 100644 index 0000000000..8176539a30 --- /dev/null +++ b/backend/.sqlx/query-4e5273b9ce05f6ee2dfd5f14c8574a0cf43682480452f7dbe23012320fe7fe25.json @@ -0,0 +1,58 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT \n tag, \n script_lang AS \"script_lang!: _\"\n FROM \n v2_job\n WHERE \n id = $1\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "tag", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "script_lang!: _", + "type_info": { + "Custom": { + "name": "script_lang", + "kind": { + "Enum": [ + "python3", + "deno", + "go", + "bash", + "postgresql", + "nativets", + "bun", + "mysql", + "bigquery", + "snowflake", + "graphql", + "powershell", + "mssql", + "php", + "bunnative", + "rust", + "ansible", + "csharp", + "oracledb", + "nu", + "java", + "duckdb" + ] + } + } + } + } + ], + "parameters": { + "Left": [ + "Uuid" + ] + }, + "nullable": [ + false, + true + ] + }, + "hash": "4e5273b9ce05f6ee2dfd5f14c8574a0cf43682480452f7dbe23012320fe7fe25" +} diff --git a/backend/.sqlx/query-4ee0017771f46f0272817d18edb821940cb5064e3f155b9630b131c09c9dba13.json b/backend/.sqlx/query-4ee0017771f46f0272817d18edb821940cb5064e3f155b9630b131c09c9dba13.json deleted file mode 100644 index 52136dd33e..0000000000 --- a/backend/.sqlx/query-4ee0017771f46f0272817d18edb821940cb5064e3f155b9630b131c09c9dba13.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n SELECT \n slot_name,\n active\n FROM\n pg_replication_slots \n WHERE \n plugin = 'pgoutput' AND\n slot_type = 'logical';\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "slot_name", - "type_info": "Name" - }, - { - "ordinal": 1, - "name": "active", - "type_info": "Bool" - } - ], - "parameters": { - "Left": [] - }, - "nullable": [ - true, - true - ] - }, - "hash": "4ee0017771f46f0272817d18edb821940cb5064e3f155b9630b131c09c9dba13" -} diff --git a/backend/.sqlx/query-553108ba3c0b8d579800bc8b5a4f887d79fb4c13b60b19c4913a8db18521958c.json b/backend/.sqlx/query-553108ba3c0b8d579800bc8b5a4f887d79fb4c13b60b19c4913a8db18521958c.json deleted file mode 100644 index e3d2e205f1..0000000000 --- a/backend/.sqlx/query-553108ba3c0b8d579800bc8b5a4f887d79fb4c13b60b19c4913a8db18521958c.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE v2_job_queue q SET suspend = 0\n FROM v2_job j, v2_job_status f\n WHERE parent_job = $1\n AND f.id = j.id AND q.id = j.id\n AND suspend = $2 AND (f.flow_status->'step')::int = 0", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Uuid", - "Int4" - ] - }, - "nullable": [] - }, - "hash": "553108ba3c0b8d579800bc8b5a4f887d79fb4c13b60b19c4913a8db18521958c" -} diff --git a/backend/.sqlx/query-56b2326015fde12b1a4efa226518566101dd27a0f3363884781071d417f8b7e7.json b/backend/.sqlx/query-56b2326015fde12b1a4efa226518566101dd27a0f3363884781071d417f8b7e7.json new file mode 100644 index 0000000000..9da0554266 --- /dev/null +++ b/backend/.sqlx/query-56b2326015fde12b1a4efa226518566101dd27a0f3363884781071d417f8b7e7.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO usage (id, is_workspace, month_, usage)\n VALUES ($1, FALSE, EXTRACT(YEAR FROM current_date) * 12 + EXTRACT(MONTH FROM current_date), 1)\n ON CONFLICT (id, is_workspace, month_) DO UPDATE SET usage = usage.usage + 1 \n RETURNING usage.usage", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "usage", + "type_info": "Int4" + } + ], + "parameters": { + "Left": [ + "Varchar" + ] + }, + "nullable": [ + false + ] + }, + "hash": "56b2326015fde12b1a4efa226518566101dd27a0f3363884781071d417f8b7e7" +} diff --git a/backend/.sqlx/query-61c29d684e8e683e839a6d7210b3b9b96854e5bfd752e45922c358c42ebea0c4.json b/backend/.sqlx/query-61c29d684e8e683e839a6d7210b3b9b96854e5bfd752e45922c358c42ebea0c4.json index be472306a0..1a30ad011f 100644 --- a/backend/.sqlx/query-61c29d684e8e683e839a6d7210b3b9b96854e5bfd752e45922c358c42ebea0c4.json +++ b/backend/.sqlx/query-61c29d684e8e683e839a6d7210b3b9b96854e5bfd752e45922c358c42ebea0c4.json @@ -61,7 +61,8 @@ "csharp", "oracledb", "nu", - "java" + "java", + "duckdb" ] } } diff --git a/backend/.sqlx/query-621e9a2a53187dac3ebed62f0d645b692815f1594bf302dbebd5f80d5d22b98e.json b/backend/.sqlx/query-621e9a2a53187dac3ebed62f0d645b692815f1594bf302dbebd5f80d5d22b98e.json deleted file mode 100644 index 6a66bb6f22..0000000000 --- a/backend/.sqlx/query-621e9a2a53187dac3ebed62f0d645b692815f1594bf302dbebd5f80d5d22b98e.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "INSERT INTO usage (id, is_workspace, month_, usage)\n VALUES ($1, TRUE, EXTRACT(YEAR FROM current_date) * 12 + EXTRACT(MONTH FROM current_date), 1)\n ON CONFLICT (id, is_workspace, month_) DO UPDATE SET usage = usage.usage + 1 \n RETURNING usage.usage", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "usage", - "type_info": "Int4" - } - ], - "parameters": { - "Left": [ - "Varchar" - ] - }, - "nullable": [ - false - ] - }, - "hash": "621e9a2a53187dac3ebed62f0d645b692815f1594bf302dbebd5f80d5d22b98e" -} diff --git a/backend/.sqlx/query-6a19c440a7a8064f3969cf6f48adea0bfdb683de9555e374ce5731e0b3c379f9.json b/backend/.sqlx/query-6a19c440a7a8064f3969cf6f48adea0bfdb683de9555e374ce5731e0b3c379f9.json new file mode 100644 index 0000000000..4ea7dabd04 --- /dev/null +++ b/backend/.sqlx/query-6a19c440a7a8064f3969cf6f48adea0bfdb683de9555e374ce5731e0b3c379f9.json @@ -0,0 +1,24 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT COUNT(*) FROM gcp_trigger WHERE script_path = $1 AND is_flow = $2 AND workspace_id = $3", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "count", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text", + "Bool", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "6a19c440a7a8064f3969cf6f48adea0bfdb683de9555e374ce5731e0b3c379f9" +} diff --git a/backend/.sqlx/query-6e2564dc37ee967c634deade67ceabc4c516418e595dee9cd7d651acc1f4af6e.json b/backend/.sqlx/query-6e2564dc37ee967c634deade67ceabc4c516418e595dee9cd7d651acc1f4af6e.json index 1ab5672e7d..39ebee28b4 100644 --- a/backend/.sqlx/query-6e2564dc37ee967c634deade67ceabc4c516418e595dee9cd7d651acc1f4af6e.json +++ b/backend/.sqlx/query-6e2564dc37ee967c634deade67ceabc4c516418e595dee9cd7d651acc1f4af6e.json @@ -147,7 +147,8 @@ "csharp", "oracledb", "nu", - "java" + "java", + "duckdb" ] } } diff --git a/backend/.sqlx/query-6f56acb985aa7141ea1891d7ad58a32c35d1b02fe7070c92a2e62c1a5339c396.json b/backend/.sqlx/query-6f56acb985aa7141ea1891d7ad58a32c35d1b02fe7070c92a2e62c1a5339c396.json deleted file mode 100644 index f3bda55eec..0000000000 --- a/backend/.sqlx/query-6f56acb985aa7141ea1891d7ad58a32c35d1b02fe7070c92a2e62c1a5339c396.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT slot_name FROM pg_replication_slots where slot_name = $1", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "slot_name", - "type_info": "Name" - } - ], - "parameters": { - "Left": [ - "Name" - ] - }, - "nullable": [ - true - ] - }, - "hash": "6f56acb985aa7141ea1891d7ad58a32c35d1b02fe7070c92a2e62c1a5339c396" -} diff --git a/backend/.sqlx/query-7e64ba7e2362cc19d2aed9f34c9879983922e96a9baab7c1a2b09ed2b1c261e2.json b/backend/.sqlx/query-7e64ba7e2362cc19d2aed9f34c9879983922e96a9baab7c1a2b09ed2b1c261e2.json deleted file mode 100644 index 9984af6e6e..0000000000 --- a/backend/.sqlx/query-7e64ba7e2362cc19d2aed9f34c9879983922e96a9baab7c1a2b09ed2b1c261e2.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT \n active_pid \n FROM \n pg_replication_slots \n WHERE \n slot_name = $1\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "active_pid", - "type_info": "Int4" - } - ], - "parameters": { - "Left": [ - "Name" - ] - }, - "nullable": [ - true - ] - }, - "hash": "7e64ba7e2362cc19d2aed9f34c9879983922e96a9baab7c1a2b09ed2b1c261e2" -} diff --git a/backend/.sqlx/query-804fc11e35f4afc0db194b6fe2594f91df7e588d4d2431bc85f4d8734920c8bf.json b/backend/.sqlx/query-804fc11e35f4afc0db194b6fe2594f91df7e588d4d2431bc85f4d8734920c8bf.json index 89745202b0..7933311a4c 100644 --- a/backend/.sqlx/query-804fc11e35f4afc0db194b6fe2594f91df7e588d4d2431bc85f4d8734920c8bf.json +++ b/backend/.sqlx/query-804fc11e35f4afc0db194b6fe2594f91df7e588d4d2431bc85f4d8734920c8bf.json @@ -34,7 +34,8 @@ "csharp", "oracledb", "nu", - "java" + "java", + "duckdb" ] } } diff --git a/backend/.sqlx/query-83f64dd93b1ddc03b84681d65d9be69959987cbac1d83b64225fd1bf9ab047c9.json b/backend/.sqlx/query-83f64dd93b1ddc03b84681d65d9be69959987cbac1d83b64225fd1bf9ab047c9.json new file mode 100644 index 0000000000..3ca69e27dd --- /dev/null +++ b/backend/.sqlx/query-83f64dd93b1ddc03b84681d65d9be69959987cbac1d83b64225fd1bf9ab047c9.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO usage (id, is_workspace, month_, usage)\n VALUES ($1, TRUE, EXTRACT(YEAR FROM current_date) * 12 + EXTRACT(MONTH FROM current_date), 1)\n ON CONFLICT (id, is_workspace, month_) DO UPDATE SET usage = usage.usage + 1 \n RETURNING usage.usage", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "usage", + "type_info": "Int4" + } + ], + "parameters": { + "Left": [ + "Varchar" + ] + }, + "nullable": [ + false + ] + }, + "hash": "83f64dd93b1ddc03b84681d65d9be69959987cbac1d83b64225fd1bf9ab047c9" +} diff --git a/backend/.sqlx/query-86ae16175ace0179e784aacfd381771f0137ecab6671d632febadede729e7783.json b/backend/.sqlx/query-86ae16175ace0179e784aacfd381771f0137ecab6671d632febadede729e7783.json deleted file mode 100644 index cbf85f6b0d..0000000000 --- a/backend/.sqlx/query-86ae16175ace0179e784aacfd381771f0137ecab6671d632febadede729e7783.json +++ /dev/null @@ -1,40 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n SELECT\n puballtables AS all_table,\n pubinsert AS insert,\n pubupdate AS update,\n pubdelete AS delete\n FROM\n pg_publication\n WHERE\n pubname = $1\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "all_table", - "type_info": "Bool" - }, - { - "ordinal": 1, - "name": "insert", - "type_info": "Bool" - }, - { - "ordinal": 2, - "name": "update", - "type_info": "Bool" - }, - { - "ordinal": 3, - "name": "delete", - "type_info": "Bool" - } - ], - "parameters": { - "Left": [ - "Name" - ] - }, - "nullable": [ - false, - false, - false, - false - ] - }, - "hash": "86ae16175ace0179e784aacfd381771f0137ecab6671d632febadede729e7783" -} diff --git a/backend/.sqlx/query-997586ac14384db2c0eeee1bb3382cc6ae013695d0cda6da9ab848ca1b9a9606.json b/backend/.sqlx/query-997586ac14384db2c0eeee1bb3382cc6ae013695d0cda6da9ab848ca1b9a9606.json new file mode 100644 index 0000000000..1edf26100f --- /dev/null +++ b/backend/.sqlx/query-997586ac14384db2c0eeee1bb3382cc6ae013695d0cda6da9ab848ca1b9a9606.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE worker_ping SET\nping_at = now(),\njobs_executed = 1,\ncurrent_job_id = $1,\ncurrent_job_workspace_id = 'admins'\nWHERE worker = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Uuid", + "Text" + ] + }, + "nullable": [] + }, + "hash": "997586ac14384db2c0eeee1bb3382cc6ae013695d0cda6da9ab848ca1b9a9606" +} diff --git a/backend/.sqlx/query-a8b470b463ca4b7c00c7ef6e9f36c23f8bbcefc288a56d61122bfd6fe5ca7e8d.json b/backend/.sqlx/query-a8b470b463ca4b7c00c7ef6e9f36c23f8bbcefc288a56d61122bfd6fe5ca7e8d.json new file mode 100644 index 0000000000..6ac4ee6755 --- /dev/null +++ b/backend/.sqlx/query-a8b470b463ca4b7c00c7ef6e9f36c23f8bbcefc288a56d61122bfd6fe5ca7e8d.json @@ -0,0 +1,24 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT COUNT(*) FROM mqtt_trigger WHERE script_path = $1 AND is_flow = $2 AND workspace_id = $3", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "count", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text", + "Bool", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "a8b470b463ca4b7c00c7ef6e9f36c23f8bbcefc288a56d61122bfd6fe5ca7e8d" +} diff --git a/backend/.sqlx/query-a8bcae108af1eda6efe3a4b8c6f8807bc464a81c0883e68f5a69b89b94b0b34b.json b/backend/.sqlx/query-a8bcae108af1eda6efe3a4b8c6f8807bc464a81c0883e68f5a69b89b94b0b34b.json index a2a19a6c35..a3cb5a82b9 100644 --- a/backend/.sqlx/query-a8bcae108af1eda6efe3a4b8c6f8807bc464a81c0883e68f5a69b89b94b0b34b.json +++ b/backend/.sqlx/query-a8bcae108af1eda6efe3a4b8c6f8807bc464a81c0883e68f5a69b89b94b0b34b.json @@ -36,7 +36,8 @@ "csharp", "oracledb", "nu", - "java" + "java", + "duckdb" ] } } diff --git a/backend/.sqlx/query-a9db7b2f435bb82acb8c5eeb7f800b28f3256491fdaa168591adc7b4b9f3327a.json b/backend/.sqlx/query-a9db7b2f435bb82acb8c5eeb7f800b28f3256491fdaa168591adc7b4b9f3327a.json index 8cc09349ec..16fea8720d 100644 --- a/backend/.sqlx/query-a9db7b2f435bb82acb8c5eeb7f800b28f3256491fdaa168591adc7b4b9f3327a.json +++ b/backend/.sqlx/query-a9db7b2f435bb82acb8c5eeb7f800b28f3256491fdaa168591adc7b4b9f3327a.json @@ -41,7 +41,8 @@ "csharp", "oracledb", "nu", - "java" + "java", + "duckdb" ] } } diff --git a/backend/.sqlx/query-ab04cda71f8e2be9acbecabe1ee5ef756b8e5c1955fbe111df9ee171dc262338.json b/backend/.sqlx/query-ab04cda71f8e2be9acbecabe1ee5ef756b8e5c1955fbe111df9ee171dc262338.json index 47e4f47005..c8cbb8e10d 100644 --- a/backend/.sqlx/query-ab04cda71f8e2be9acbecabe1ee5ef756b8e5c1955fbe111df9ee171dc262338.json +++ b/backend/.sqlx/query-ab04cda71f8e2be9acbecabe1ee5ef756b8e5c1955fbe111df9ee171dc262338.json @@ -65,7 +65,8 @@ "csharp", "oracledb", "nu", - "java" + "java", + "duckdb" ] } } diff --git a/backend/.sqlx/query-baa1dddc616419bf4b923715f0a863bc0ff69c98db0f0c8f55e4ac89fdde7a60.json b/backend/.sqlx/query-baa1dddc616419bf4b923715f0a863bc0ff69c98db0f0c8f55e4ac89fdde7a60.json deleted file mode 100644 index e772354ea8..0000000000 --- a/backend/.sqlx/query-baa1dddc616419bf4b923715f0a863bc0ff69c98db0f0c8f55e4ac89fdde7a60.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT pubname FROM pg_publication WHERE pubname = $1", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "pubname", - "type_info": "Name" - } - ], - "parameters": { - "Left": [ - "Name" - ] - }, - "nullable": [ - false - ] - }, - "hash": "baa1dddc616419bf4b923715f0a863bc0ff69c98db0f0c8f55e4ac89fdde7a60" -} diff --git a/backend/.sqlx/query-c3025cdb6e421e1225d420e8b1efd18d1dd3bb2fac53c1f2df648b61fb7488aa.json b/backend/.sqlx/query-c3025cdb6e421e1225d420e8b1efd18d1dd3bb2fac53c1f2df648b61fb7488aa.json deleted file mode 100644 index 638417aabb..0000000000 --- a/backend/.sqlx/query-c3025cdb6e421e1225d420e8b1efd18d1dd3bb2fac53c1f2df648b61fb7488aa.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE worker_ping SET \nping_at = now(), \njobs_executed = 1, \ncurrent_job_id = $1, \ncurrent_job_workspace_id = 'admins' \nWHERE worker = $2", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Uuid", - "Text" - ] - }, - "nullable": [] - }, - "hash": "c3025cdb6e421e1225d420e8b1efd18d1dd3bb2fac53c1f2df648b61fb7488aa" -} diff --git a/backend/.sqlx/query-ca3ba808e020c8c7a35eaef842b20cfeee64fd47ded72fce55cc75e0bbb291a8.json b/backend/.sqlx/query-ca3ba808e020c8c7a35eaef842b20cfeee64fd47ded72fce55cc75e0bbb291a8.json new file mode 100644 index 0000000000..6d44fb2840 --- /dev/null +++ b/backend/.sqlx/query-ca3ba808e020c8c7a35eaef842b20cfeee64fd47ded72fce55cc75e0bbb291a8.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO usage (id, is_workspace, month_, usage) \n VALUES ($1, TRUE, EXTRACT(YEAR FROM current_date) * 12 + EXTRACT(MONTH FROM current_date), $2) \n ON CONFLICT (id, is_workspace, month_) DO UPDATE SET usage = usage.usage + $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Int4" + ] + }, + "nullable": [] + }, + "hash": "ca3ba808e020c8c7a35eaef842b20cfeee64fd47ded72fce55cc75e0bbb291a8" +} diff --git a/backend/.sqlx/query-cccdcb7fe7968eadfc04d8957a8e98b2f2d92a6d7f687a9dd5a70edb3d5a63e6.json b/backend/.sqlx/query-cccdcb7fe7968eadfc04d8957a8e98b2f2d92a6d7f687a9dd5a70edb3d5a63e6.json index 412b64c480..94f591b3da 100644 --- a/backend/.sqlx/query-cccdcb7fe7968eadfc04d8957a8e98b2f2d92a6d7f687a9dd5a70edb3d5a63e6.json +++ b/backend/.sqlx/query-cccdcb7fe7968eadfc04d8957a8e98b2f2d92a6d7f687a9dd5a70edb3d5a63e6.json @@ -70,7 +70,8 @@ "csharp", "oracledb", "nu", - "java" + "java", + "duckdb" ] } } diff --git a/backend/.sqlx/query-d15f02f090b8d1a7e816fe11b2e0867540ab6bb02ac6bf82decc220dce0ab048.json b/backend/.sqlx/query-d15f02f090b8d1a7e816fe11b2e0867540ab6bb02ac6bf82decc220dce0ab048.json index 0a2b8c3a0c..3a23232dcc 100644 --- a/backend/.sqlx/query-d15f02f090b8d1a7e816fe11b2e0867540ab6bb02ac6bf82decc220dce0ab048.json +++ b/backend/.sqlx/query-d15f02f090b8d1a7e816fe11b2e0867540ab6bb02ac6bf82decc220dce0ab048.json @@ -42,7 +42,8 @@ "csharp", "oracledb", "nu", - "java" + "java", + "duckdb" ] } } diff --git a/backend/.sqlx/query-e38240e6d50bfe60e1c2b649588eb41dcef121ed161db04b2568ac2d990aed7c.json b/backend/.sqlx/query-e38240e6d50bfe60e1c2b649588eb41dcef121ed161db04b2568ac2d990aed7c.json deleted file mode 100644 index 1ff35d7227..0000000000 --- a/backend/.sqlx/query-e38240e6d50bfe60e1c2b649588eb41dcef121ed161db04b2568ac2d990aed7c.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "INSERT INTO usage (id, is_workspace, month_, usage) \n VALUES ($1, FALSE, EXTRACT(YEAR FROM current_date) * 12 + EXTRACT(MONTH FROM current_date), $2) \n ON CONFLICT (id, is_workspace, month_) DO UPDATE SET usage = usage.usage + $2", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Varchar", - "Int4" - ] - }, - "nullable": [] - }, - "hash": "e38240e6d50bfe60e1c2b649588eb41dcef121ed161db04b2568ac2d990aed7c" -} diff --git a/backend/.sqlx/query-f1dbcb6e6d82d17c19eb88c0e67dc1cb8baf5bd40b75a2a9cd3ebac440fda632.json b/backend/.sqlx/query-f1dbcb6e6d82d17c19eb88c0e67dc1cb8baf5bd40b75a2a9cd3ebac440fda632.json new file mode 100644 index 0000000000..b58fad894b --- /dev/null +++ b/backend/.sqlx/query-f1dbcb6e6d82d17c19eb88c0e67dc1cb8baf5bd40b75a2a9cd3ebac440fda632.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE v2_job_queue q SET suspend = 0\n FROM v2_job j, v2_job_status f\n WHERE q.workspace_id = $1 AND q.suspend = $3 AND j.parent_job = $2\n AND f.id = j.id AND q.id = j.id\n AND (f.flow_status->'step')::int = 0", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Uuid", + "Int4" + ] + }, + "nullable": [] + }, + "hash": "f1dbcb6e6d82d17c19eb88c0e67dc1cb8baf5bd40b75a2a9cd3ebac440fda632" +} diff --git a/backend/.sqlx/query-f22964772dc2d67aee437bbbd08b64792c00da1d713d7ca8f9904ccce7bfdae7.json b/backend/.sqlx/query-f22964772dc2d67aee437bbbd08b64792c00da1d713d7ca8f9904ccce7bfdae7.json index b3a352de55..b400be3485 100644 --- a/backend/.sqlx/query-f22964772dc2d67aee437bbbd08b64792c00da1d713d7ca8f9904ccce7bfdae7.json +++ b/backend/.sqlx/query-f22964772dc2d67aee437bbbd08b64792c00da1d713d7ca8f9904ccce7bfdae7.json @@ -147,7 +147,8 @@ "csharp", "oracledb", "nu", - "java" + "java", + "duckdb" ] } } diff --git a/backend/.sqlx/query-fd5754fe3c6346ae28818a9d60d144a40f8884f47e5bbdd2824e939dafd8f154.json b/backend/.sqlx/query-fd5754fe3c6346ae28818a9d60d144a40f8884f47e5bbdd2824e939dafd8f154.json deleted file mode 100644 index e797ab8805..0000000000 --- a/backend/.sqlx/query-fd5754fe3c6346ae28818a9d60d144a40f8884f47e5bbdd2824e939dafd8f154.json +++ /dev/null @@ -1,40 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n SELECT\n schemaname AS schema_name,\n tablename AS table_name,\n CASE\n WHEN array_length(attnames, 1) = (SELECT COUNT(*) FROM information_schema.columns WHERE table_schema = pg_publication_tables.schemaname AND table_name = pg_publication_tables.tablename)\n THEN NULL\n ELSE attnames\n END AS columns,\n rowfilter AS where_clause\n FROM\n pg_publication_tables\n WHERE\n pubname = $1;\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "schema_name", - "type_info": "Name" - }, - { - "ordinal": 1, - "name": "table_name", - "type_info": "Name" - }, - { - "ordinal": 2, - "name": "columns", - "type_info": "NameArray" - }, - { - "ordinal": 3, - "name": "where_clause", - "type_info": "Text" - } - ], - "parameters": { - "Left": [ - "Name" - ] - }, - "nullable": [ - true, - true, - null, - true - ] - }, - "hash": "fd5754fe3c6346ae28818a9d60d144a40f8884f47e5bbdd2824e939dafd8f154" -} diff --git a/backend/.sqlx/query-ff0403790674cdb07022af71c2377afbd8b3a660b3be27514b517c077c63c238.json b/backend/.sqlx/query-ff0403790674cdb07022af71c2377afbd8b3a660b3be27514b517c077c63c238.json index c7644fa79a..4e911403de 100644 --- a/backend/.sqlx/query-ff0403790674cdb07022af71c2377afbd8b3a660b3be27514b517c077c63c238.json +++ b/backend/.sqlx/query-ff0403790674cdb07022af71c2377afbd8b3a660b3be27514b517c077c63c238.json @@ -65,7 +65,8 @@ "csharp", "oracledb", "nu", - "java" + "java", + "duckdb" ] } } diff --git a/backend/CLAUDE.md b/backend/CLAUDE.md deleted file mode 100644 index 382c7a04a9..0000000000 --- a/backend/CLAUDE.md +++ /dev/null @@ -1,104 +0,0 @@ -# Windmill Backend - Rust Best Practices - -## Project Structure - -Windmill uses a workspace-based architecture with multiple crates: - -- **windmill-api**: API server functionality -- **windmill-worker**: Job execution -- **windmill-common**: Shared code used by all crates -- **windmill-queue**: Job & flow queuing -- **windmill-audit**: Audit logging -- Other specialized crates (git-sync, autoscaling, etc.) - -## Adding New Code - -### Module Organization - -- Place new code in the appropriate crate based on functionality -- For API endpoints, create or modify files in `windmill-api/src/` organized by domain -- For shared functionality, use `windmill-common/src/` -- Use the `_ee.rs` suffix for enterprise-only modules -- Follow existing patterns for file structure and organization - -### Error Handling - -- Use the custom `Error` enum from `windmill-common::error` -- Return `Result` or `JsonResult` for functions that can fail -- Use the `?` operator for error propagation -- Add location tracking to errors using `#[track_caller]` - -### Database Operations - -- Use `sqlx` for database operations with prepared statements -- Leverage existing database helper functions in `db.rs` modules -- Use transactions for multi-step operations -- Handle database errors properly - -### API Endpoints - -- Follow existing patterns in the `windmill-api` crate -- Use axum's routing system and extractors -- Group related routes together -- Use consistent response formats (JSON) -- Follow proper authentication and authorization patterns - -## Performance Optimizations - -When generating code, especially involving `serde`, `sqlx`, and `tokio`, prioritize performance by applying the following principles: - -### Serde Optimizations (Serialization & Deserialization) - -- **Specify Structure Explicitly:** When defining structs for Serde (`#[derive(Serialize, Deserialize)]`), use `#[serde(...` attributes extensively. This includes: - - `#[serde(rename = "...")]` or `#[serde(alias = "...")]` to map external names precisely, avoiding dynamic lookups. - - `#[serde(default)]` for optional fields with default values, reducing parsing complexity. - - `#[serde(skip_serializing_if = "...")]` to avoid writing fields that meet a certain condition (e.g., `Option::is_none()`, `Vec::is_empty()`, or a custom function), reducing output size and serialization work. - - `#[serde(skip_serializing)]` or `#[serde(skip_deserializing)]` for fields that should _not_ be included. -- **Prefer Borrowing:** Where possible and safe (data lifetime allows), use `Cow<'a, str>` or `&'a str` (with `#[serde(borrow)]`) instead of `String` for string fields during deserialization. This avoids allocating new strings, enabling zero-copy reading from the input buffer. Apply this principle to byte slices (`&'a [u8]` / `Cow<'a, [u8]>`) and potentially borrowed vectors as well. -- **Avoid Intermediate `Value`:** Unless the data structure is truly dynamic or unknown at compile time, deserialize directly into a well-defined struct or enum rather than into `serde_json::Value` (or equivalent for other formats). This avoids unnecessary heap allocations and type switching. - -### SQLx Optimizations (Database Interaction) - -- **Select Only Necessary Columns:** In `SELECT` queries, list specific column names rather than using `SELECT *`. This reduces data transferred from the database and the work needed for hydration/deserialization. -- **Batch Operations:** For multiple `INSERT`, `UPDATE`, or `DELETE` statements, prefer executing them in a single query if the database and driver support it efficiently (e.g., `INSERT INTO ... VALUES (...), (...), ...`). This minimizes round trips to the database. -- **Avoid N+1 Queries:** Do not loop through results of one query and execute a separate query for each item (e.g., fetching users, then querying for each user's profile in a loop). Instead, use JOINs or a single query with an `IN` clause to fetch related data efficiently. -- **Deserialize Directly:** Use `#[derive(FromRow)]` on structs and ensure the struct fields match the selected columns in the query. This allows SQLx to hydrate objects directly, avoiding intermediate data structures. -- **Parameterize Queries:** Always use SQLx's query methods (`.bind(...)`) to pass values as parameters rather than string formatting. This prevents SQL injection and allows the database to cache query plans, improving performance on repeated executions. - -### Tokio Optimizations (Asynchronous Runtime) - -- **Avoid Blocking Operations:** **Crucially**, never perform blocking operations (synchronous file I/O, `std::thread::sleep`, CPU-bound loops, `std::sync::Mutex::lock`, blocking network calls without `tokio::net`) directly within an `async fn` or a standard `tokio::spawn` task. Blocking pauses the entire worker thread, potentially starving other tasks. Use `tokio::task::spawn_blocking` for CPU-intensive work or blocking I/O. -- **Use Tokio's Async Primitives:** Prefer `tokio::sync` (channels, mutexes, semaphores), `tokio::io`, `tokio::net`, and `tokio::time` over their `std` counterparts in asynchronous contexts. These are designed to yield control back to the scheduler. -- **Manage Concurrency:** Be mindful of how many tasks are spawned. Creating a new task for every tiny piece of work can introduce overhead. Group related asynchronous operations where appropriate. -- **Handle Shared State Efficiently:** Use `Arc` for shared ownership in concurrent tasks. When shared state needs mutation, prefer `tokio::sync::Mutex` over `std::sync::Mutex` in `async` code. Consider `tokio::sync::RwLock` if reads significantly outnumber writes. Minimize the duration for which locks are held. -- **Understand `.await`:** Place `.await` strategically to allow the runtime to switch to other ready tasks. Ensure that `.await` points to genuinely asynchronous operations. -- **Backpressure:** If dealing with data streams or queues between tasks, implement backpressure mechanisms (e.g., bounded channels like `tokio::sync::mpsc::channel`) to prevent one component from overwhelming another or critical resources like the database. - -## Enterprise Features - -- Use feature flags for enterprise functionality -- Conditionally compile with `#[cfg(feature = "enterprise")]` -- Isolate enterprise code in separate modules - -## Code Style - -- Group imports by external and internal crates -- Place struct/enum definitions before implementations -- Group similar functionality together -- Use descriptive naming consistent with the codebase -- Follow existing patterns for async code using tokio - -## Testing - -- Write unit tests for core functionality -- Use the `#[cfg(test)]` module for test code -- For database tests, use the existing test utilities - -## Common Crates Used - -- **tokio**: For async runtime -- **axum**: For web server and routing -- **sqlx**: For database operations -- **serde**: For serialization/deserialization -- **tracing**: For logging and diagnostics -- **reqwest**: For HTTP client functionality diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 445dad0222..edaea2c45a 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -175,9 +175,9 @@ dependencies = [ [[package]] name = "anstream" -version = "0.6.18" +version = "0.6.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8acc5369981196006228e28809f761875c0327210a891e941f4c683b3a99529b" +checksum = "301af1932e46185686725e0fad2f8f2aa7da69dd70bf6ecc44d6b703844a3933" dependencies = [ "anstyle", "anstyle-parse", @@ -190,33 +190,33 @@ dependencies = [ [[package]] name = "anstyle" -version = "1.0.10" +version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "55cc3b69f167a1ef2e161439aa98aed94e6028e5f9a59be9a6ffb47aef1651f9" +checksum = "862ed96ca487e809f1c8e5a8447f6ee2cf102f846893800b20cebdf541fc6bbd" [[package]] name = "anstyle-parse" -version = "0.2.6" +version = "0.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b2d16507662817a6a20a9ea92df6652ee4f94f914589377d69f3b21bc5798a9" +checksum = "4e7644824f0aa2c7b9384579234ef10eb7efb6a0deb83f9630a49594dd9c15c2" dependencies = [ "utf8parse", ] [[package]] name = "anstyle-query" -version = "1.1.2" +version = "1.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79947af37f4177cfead1110013d678905c37501914fba0efea834c3fe9a8d60c" +checksum = "6c8bdeb6047d8983be085bab0ba1472e6dc604e7041dbf6fcd5e71523014fae9" dependencies = [ "windows-sys 0.59.0", ] [[package]] name = "anstyle-wincon" -version = "3.0.8" +version = "3.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6680de5231bd6ee4c6191b8a1325daa282b415391ec9d3a37bd34f2060dc73fa" +checksum = "403f75924867bb1033c59fbf0797484329750cfbe3c4325cd33127941fabc882" dependencies = [ "anstyle", "once_cell_polyfill", @@ -286,57 +286,25 @@ dependencies = [ "serde", ] -[[package]] -name = "arrow" -version = "54.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5ec52ba94edeed950e4a41f75d35376df196e8cb04437f7280a5aa49f20f796" -dependencies = [ - "arrow-arith 54.3.1", - "arrow-array 54.3.1", - "arrow-buffer 54.3.1", - "arrow-cast 54.3.1", - "arrow-data 54.3.1", - "arrow-ord 54.3.1", - "arrow-row 54.3.1", - "arrow-schema 54.3.1", - "arrow-select 54.3.1", - "arrow-string 54.3.1", -] - [[package]] name = "arrow" version = "55.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b1bb018b6960c87fd9d025009820406f74e83281185a8bdcb44880d2aa5c9a87" dependencies = [ - "arrow-arith 55.1.0", - "arrow-array 55.1.0", - "arrow-buffer 55.1.0", - "arrow-cast 55.1.0", + "arrow-arith", + "arrow-array", + "arrow-buffer", + "arrow-cast", "arrow-csv", - "arrow-data 55.1.0", + "arrow-data", "arrow-ipc", "arrow-json", - "arrow-ord 55.1.0", - "arrow-row 55.1.0", - "arrow-schema 55.1.0", - "arrow-select 55.1.0", - "arrow-string 55.1.0", -] - -[[package]] -name = "arrow-arith" -version = "54.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fc766fdacaf804cb10c7c70580254fcdb5d55cdfda2bc57b02baf5223a3af9e" -dependencies = [ - "arrow-array 54.3.1", - "arrow-buffer 54.3.1", - "arrow-data 54.3.1", - "arrow-schema 54.3.1", - "chrono", - "num", + "arrow-ord", + "arrow-row", + "arrow-schema", + "arrow-select", + "arrow-string", ] [[package]] @@ -345,30 +313,14 @@ version = "55.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "44de76b51473aa888ecd6ad93ceb262fb8d40d1f1154a4df2f069b3590aa7575" dependencies = [ - "arrow-array 55.1.0", - "arrow-buffer 55.1.0", - "arrow-data 55.1.0", - "arrow-schema 55.1.0", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", "chrono", "num", ] -[[package]] -name = "arrow-array" -version = "54.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a12fcdb3f1d03f69d3ec26ac67645a8fe3f878d77b5ebb0b15d64a116c212985" -dependencies = [ - "ahash 0.8.12", - "arrow-buffer 54.3.1", - "arrow-data 54.3.1", - "arrow-schema 54.3.1", - "chrono", - "half", - "hashbrown 0.15.3", - "num", -] - [[package]] name = "arrow-array" version = "55.1.0" @@ -376,24 +328,13 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "29ed77e22744475a9a53d00026cf8e166fe73cf42d89c4c4ae63607ee1cfcc3f" dependencies = [ "ahash 0.8.12", - "arrow-buffer 55.1.0", - "arrow-data 55.1.0", - "arrow-schema 55.1.0", + "arrow-buffer", + "arrow-data", + "arrow-schema", "chrono", "chrono-tz", "half", - "hashbrown 0.15.3", - "num", -] - -[[package]] -name = "arrow-buffer" -version = "54.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "263f4801ff1839ef53ebd06f99a56cecd1dbaf314ec893d93168e2e860e0291c" -dependencies = [ - "bytes", - "half", + "hashbrown 0.15.4", "num", ] @@ -408,38 +349,17 @@ dependencies = [ "num", ] -[[package]] -name = "arrow-cast" -version = "54.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ede6175fbc039dfc946a61c1b6d42fd682fcecf5ab5d148fbe7667705798cac9" -dependencies = [ - "arrow-array 54.3.1", - "arrow-buffer 54.3.1", - "arrow-data 54.3.1", - "arrow-schema 54.3.1", - "arrow-select 54.3.1", - "atoi", - "base64 0.22.1", - "chrono", - "comfy-table", - "half", - "lexical-core", - "num", - "ryu", -] - [[package]] name = "arrow-cast" version = "55.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f39e1d774ece9292697fcbe06b5584401b26bd34be1bec25c33edae65c2420ff" dependencies = [ - "arrow-array 55.1.0", - "arrow-buffer 55.1.0", - "arrow-data 55.1.0", - "arrow-schema 55.1.0", - "arrow-select 55.1.0", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", "atoi", "base64 0.22.1", "chrono", @@ -456,9 +376,9 @@ version = "55.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9055c972a07bf12c2a827debfd34f88d3b93da1941d36e1d9fee85eebe38a12a" dependencies = [ - "arrow-array 55.1.0", - "arrow-cast 55.1.0", - "arrow-schema 55.1.0", + "arrow-array", + "arrow-cast", + "arrow-schema", "chrono", "csv", "csv-core", @@ -466,26 +386,14 @@ dependencies = [ "regex", ] -[[package]] -name = "arrow-data" -version = "54.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61cfdd7d99b4ff618f167e548b2411e5dd2c98c0ddebedd7df433d34c20a4429" -dependencies = [ - "arrow-buffer 54.3.1", - "arrow-schema 54.3.1", - "half", - "num", -] - [[package]] name = "arrow-data" version = "55.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf75ac27a08c7f48b88e5c923f267e980f27070147ab74615ad85b5c5f90473d" dependencies = [ - "arrow-buffer 55.1.0", - "arrow-schema 55.1.0", + "arrow-buffer", + "arrow-schema", "half", "num", ] @@ -496,10 +404,10 @@ version = "55.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a222f0d93772bd058d1268f4c28ea421a603d66f7979479048c429292fac7b2e" dependencies = [ - "arrow-array 55.1.0", - "arrow-buffer 55.1.0", - "arrow-data 55.1.0", - "arrow-schema 55.1.0", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", "flatbuffers", "lz4_flex", ] @@ -510,11 +418,11 @@ version = "55.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9085342bbca0f75e8cb70513c0807cc7351f1fbf5cb98192a67d5e3044acb033" dependencies = [ - "arrow-array 55.1.0", - "arrow-buffer 55.1.0", - "arrow-cast 55.1.0", - "arrow-data 55.1.0", - "arrow-schema 55.1.0", + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-data", + "arrow-schema", "chrono", "half", "indexmap 2.9.0", @@ -526,43 +434,17 @@ dependencies = [ "simdutf8", ] -[[package]] -name = "arrow-ord" -version = "54.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0a3334a743bd2a1479dbc635540617a3923b4b2f6870f37357339e6b5363c21" -dependencies = [ - "arrow-array 54.3.1", - "arrow-buffer 54.3.1", - "arrow-data 54.3.1", - "arrow-schema 54.3.1", - "arrow-select 54.3.1", -] - [[package]] name = "arrow-ord" version = "55.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ab2f1065a5cad7b9efa9e22ce5747ce826aa3855766755d4904535123ef431e7" dependencies = [ - "arrow-array 55.1.0", - "arrow-buffer 55.1.0", - "arrow-data 55.1.0", - "arrow-schema 55.1.0", - "arrow-select 55.1.0", -] - -[[package]] -name = "arrow-row" -version = "54.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d1d7a7291d2c5107e92140f75257a99343956871f3d3ab33a7b41532f79cb68" -dependencies = [ - "arrow-array 54.3.1", - "arrow-buffer 54.3.1", - "arrow-data 54.3.1", - "arrow-schema 54.3.1", - "half", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", ] [[package]] @@ -571,40 +453,20 @@ version = "55.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3703a0e3e92d23c3f756df73d2dc9476873f873a76ae63ef9d3de17fda83b2d8" dependencies = [ - "arrow-array 55.1.0", - "arrow-buffer 55.1.0", - "arrow-data 55.1.0", - "arrow-schema 55.1.0", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", "half", ] -[[package]] -name = "arrow-schema" -version = "54.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39cfaf5e440be44db5413b75b72c2a87c1f8f0627117d110264048f2969b99e9" -dependencies = [ - "bitflags 2.9.1", -] - [[package]] name = "arrow-schema" version = "55.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "73a47aa0c771b5381de2b7f16998d351a6f4eb839f1e13d48353e17e873d969b" - -[[package]] -name = "arrow-select" -version = "54.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69efcd706420e52cd44f5c4358d279801993846d1c2a8e52111853d61d55a619" dependencies = [ - "ahash 0.8.12", - "arrow-array 54.3.1", - "arrow-buffer 54.3.1", - "arrow-data 54.3.1", - "arrow-schema 54.3.1", - "num", + "bitflags 2.9.1", ] [[package]] @@ -614,41 +476,24 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24b7b85575702b23b85272b01bc1c25a01c9b9852305e5d0078c79ba25d995d4" dependencies = [ "ahash 0.8.12", - "arrow-array 55.1.0", - "arrow-buffer 55.1.0", - "arrow-data 55.1.0", - "arrow-schema 55.1.0", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", "num", ] -[[package]] -name = "arrow-string" -version = "54.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a21546b337ab304a32cfc0770f671db7411787586b45b78b4593ae78e64e2b03" -dependencies = [ - "arrow-array 54.3.1", - "arrow-buffer 54.3.1", - "arrow-data 54.3.1", - "arrow-schema 54.3.1", - "arrow-select 54.3.1", - "memchr", - "num", - "regex", - "regex-syntax 0.8.5", -] - [[package]] name = "arrow-string" version = "55.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9260fddf1cdf2799ace2b4c2fc0356a9789fa7551e0953e35435536fecefebbd" dependencies = [ - "arrow-array 55.1.0", - "arrow-buffer 55.1.0", - "arrow-data 55.1.0", - "arrow-schema 55.1.0", - "arrow-select 55.1.0", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", "memchr", "num", "regex", @@ -791,7 +636,7 @@ dependencies = [ "bytes", "http 1.3.1", "rand 0.8.5", - "reqwest 0.12.17", + "reqwest 0.12.19", "serde", "serde-aux", "serde_json", @@ -991,9 +836,9 @@ dependencies = [ [[package]] name = "aws-sdk-sqs" -version = "1.71.0" +version = "1.72.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0519921fc7065a54b8dec1d10de2893d99a61d030b9f8eb00ee83eb62d2a2676" +checksum = "e7c66032c543428209106e3a104402289d9f18f77c7eb3de8f6e165256c42c38" dependencies = [ "aws-credential-types", "aws-runtime", @@ -1013,9 +858,9 @@ dependencies = [ [[package]] name = "aws-sdk-sso" -version = "1.71.0" +version = "1.72.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95a4fd09d6e863655d99cd2260f271c6d1030dc6bfad68e19e126d2e4c8ceb18" +checksum = "13118ad30741222f67b1a18e5071385863914da05124652b38e172d6d3d9ce31" dependencies = [ "aws-credential-types", "aws-runtime", @@ -1035,9 +880,9 @@ dependencies = [ [[package]] name = "aws-sdk-ssooidc" -version = "1.72.0" +version = "1.73.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3224ab02ebb3074467a33d57caf6fcb487ca36f3697fdd381b0428dc72380696" +checksum = "f879a8572b4683a8f84f781695bebf2f25cf11a81a2693c31fc0e0215c2c1726" dependencies = [ "aws-credential-types", "aws-runtime", @@ -1057,9 +902,9 @@ dependencies = [ [[package]] name = "aws-sdk-sts" -version = "1.72.0" +version = "1.73.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6933f189ed1255e78175fbd73fb200c0aae7240d220ed3346f567b0ddca3083" +checksum = "f1e9c3c24e36183e2f698235ed38dcfbbdff1d09b9232dc866c4be3011e0b47e" dependencies = [ "aws-credential-types", "aws-runtime", @@ -1133,13 +978,14 @@ dependencies = [ [[package]] name = "aws-smithy-http-client" -version = "1.0.2" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e44697a9bded898dcd0b1cb997430d949b87f4f8940d91023ae9062bf218250" +checksum = "073d330f94bdf1f47bb3e0f5d45dda1e372a54a553c39ab6e9646902c8c81594" dependencies = [ "aws-smithy-async", "aws-smithy-runtime-api", "aws-smithy-types", + "h2 0.3.26", "h2 0.4.10", "http 0.2.12", "http 1.3.1", @@ -1147,7 +993,7 @@ dependencies = [ "hyper 0.14.32", "hyper 1.6.0", "hyper-rustls 0.24.2", - "hyper-rustls 0.27.6", + "hyper-rustls 0.27.7", "hyper-util", "pin-project-lite", "rustls 0.21.12", @@ -1351,9 +1197,9 @@ checksum = "7b7e4c2464d97fe331d41de9d5db0def0a96f4d823b8b32a2efd503578988973" [[package]] name = "backon" -version = "1.5.0" +version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd0b50b1b78dbadd44ab18b3c794e496f3a139abb9fbc27d9c94c4eebbb96496" +checksum = "302eaff5357a264a2c42f127ecb8bac761cf99749fc3dc95677e2743991f99e7" dependencies = [ "fastrand", "gloo-timers", @@ -1426,9 +1272,9 @@ dependencies = [ [[package]] name = "base64ct" -version = "1.7.3" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89e25b6adfb930f02d1981565a6e5d9c547ac15a96606256d3b59040e5cd4ca3" +checksum = "55248b47b0caf0546f7988906588779981c43bb1bc9d0c44087278f80cdb44ba" [[package]] name = "better_scoped_tls" @@ -1793,9 +1639,9 @@ dependencies = [ [[package]] name = "bumpalo" -version = "3.17.0" +version = "3.18.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1628fb46dfa0b37568d12e5edd512553eccf6a22a78e8bde00bb4aed84d5bdbf" +checksum = "793db76d6187cd04dff33004d8e6c9cc4e05cd330500379d2394209271b4aeee" dependencies = [ "allocator-api2", ] @@ -1835,9 +1681,9 @@ dependencies = [ [[package]] name = "bytemuck" -version = "1.23.0" +version = "1.23.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9134a6ef01ce4b366b50689c94f82c14bc72bc5d0386829828a2e2752ef7958c" +checksum = "5c76a5792e44e4abe34d3abf15636779261d45a7450612059293d1d2cfc63422" dependencies = [ "bytemuck_derive", ] @@ -2013,9 +1859,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.24" +version = "1.2.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16595d3be041c03b09d08d0858631facccee9221e579704070e6e9e4915d3bc7" +checksum = "956a5e21988b87f372569b66183b78babf23ebc2e744b733e4350a752c4dafac" dependencies = [ "jobserver", "libc", @@ -2133,9 +1979,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.39" +version = "4.5.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd60e63e9be68e5fb56422e397cf9baddded06dae1d2e523401542383bc72a9f" +checksum = "40b6887a1d8685cebccf115538db5c0efe625ccac9696ad45c409d96566e910f" dependencies = [ "clap_builder", "clap_derive", @@ -2143,9 +1989,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.39" +version = "4.5.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89cc6392a1f72bbeb820d71f32108f61fdaf18bc526e1d23954168a67759ef51" +checksum = "e0c66c08ce9f0c698cbce5c0279d0bb6ac936d8674174fe48f736533b964f59e" dependencies = [ "anstream", "anstyle", @@ -2155,9 +2001,9 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.5.32" +version = "4.5.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09176aae279615badda0765c0c0b3f6ed53f4709118af73cf4655d85d1530cd7" +checksum = "d2c7947ae4cc3d851207c1adb5b5e260ff0cca11446b1d6d1423788e442257ce" dependencies = [ "heck 0.5.0", "proc-macro2", @@ -2167,9 +2013,9 @@ dependencies = [ [[package]] name = "clap_lex" -version = "0.7.4" +version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f46ad14479a25103f283c0f10005961cf086d8dc42205bb44c46ac563475dca6" +checksum = "b94f61472cee1439c0b966b47e3aca9ae07e45d070759512cd390ea2bebc6675" [[package]] name = "clipboard-win" @@ -2228,9 +2074,9 @@ checksum = "3d7b894f5411737b7867f4827955924d7c254fc9f4d91a6aad6b097804b1018b" [[package]] name = "colorchoice" -version = "1.0.3" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b63caa9aa9397e2d9480a9b13673856c78d8ac123288526c37d7839f2a86990" +checksum = "b05b61dc5112cbb17e4b6cd61790d9845d13888356391624cbe7e41efeac1e75" [[package]] name = "comfy-table" @@ -2728,7 +2574,7 @@ dependencies = [ "hashbrown 0.14.5", "lock_api", "once_cell", - "parking_lot_core 0.9.10", + "parking_lot_core 0.9.11", ] [[package]] @@ -2742,7 +2588,7 @@ dependencies = [ "hashbrown 0.14.5", "lock_api", "once_cell", - "parking_lot_core 0.9.10", + "parking_lot_core 0.9.11", ] [[package]] @@ -2763,9 +2609,9 @@ version = "47.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ffe060b978f74ab446be722adb8a274e052e005bf6dfd171caadc3abaad10080" dependencies = [ - "arrow 55.1.0", + "arrow", "arrow-ipc", - "arrow-schema 55.1.0", + "arrow-schema", "async-trait", "bytes", "bzip2", @@ -2799,7 +2645,7 @@ dependencies = [ "itertools 0.14.0", "log", "object_store", - "parking_lot 0.12.3", + "parking_lot 0.12.4", "parquet", "rand 0.8.5", "regex", @@ -2818,7 +2664,7 @@ version = "47.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "61fe34f401bd03724a1f96d12108144f8cd495a3cdda2bf5e091822fb80b7e66" dependencies = [ - "arrow 55.1.0", + "arrow", "async-trait", "dashmap 6.1.0", "datafusion-common", @@ -2834,7 +2680,7 @@ dependencies = [ "itertools 0.14.0", "log", "object_store", - "parking_lot 0.12.3", + "parking_lot 0.12.4", "tokio", ] @@ -2844,7 +2690,7 @@ version = "47.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a4411b8e3bce5e0fc7521e44f201def2e2d5d1b5f176fb56e8cdc9942c890f00" dependencies = [ - "arrow 55.1.0", + "arrow", "async-trait", "datafusion-catalog", "datafusion-common", @@ -2868,7 +2714,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0734015d81c8375eb5d4869b7f7ecccc2ee8d6cb81948ef737cd0e7b743bd69c" dependencies = [ "ahash 0.8.12", - "arrow 55.1.0", + "arrow", "arrow-ipc", "base64 0.22.1", "half", @@ -2902,7 +2748,7 @@ version = "47.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "04e602dcdf2f50c2abf297cc2203c73531e6f48b29516af7695d338cf2a778b1" dependencies = [ - "arrow 55.1.0", + "arrow", "async-compression", "async-trait", "bytes", @@ -2938,7 +2784,7 @@ version = "47.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3bb2253952dc32296ed5b84077cb2e0257fea4be6373e1c376426e17ead4ef6" dependencies = [ - "arrow 55.1.0", + "arrow", "async-trait", "bytes", "datafusion-catalog", @@ -2963,7 +2809,7 @@ version = "47.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5b8c7f47a5d2fe03bfa521ec9bafdb8a5c82de8377f60967c3663f00c8790352" dependencies = [ - "arrow 55.1.0", + "arrow", "async-trait", "bytes", "datafusion-catalog", @@ -2988,7 +2834,7 @@ version = "47.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "27d15868ea39ed2dc266728b554f6304acd473de2142281ecfa1294bb7415923" dependencies = [ - "arrow 55.1.0", + "arrow", "async-trait", "bytes", "datafusion-catalog", @@ -3007,7 +2853,7 @@ dependencies = [ "itertools 0.14.0", "log", "object_store", - "parking_lot 0.12.3", + "parking_lot 0.12.4", "parquet", "rand 0.8.5", "tokio", @@ -3025,14 +2871,14 @@ version = "47.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06f004d100f49a3658c9da6fb0c3a9b760062d96cd4ad82ccc3b7b69a9fb2f84" dependencies = [ - "arrow 55.1.0", + "arrow", "dashmap 6.1.0", "datafusion-common", "datafusion-expr", "futures", "log", "object_store", - "parking_lot 0.12.3", + "parking_lot 0.12.4", "rand 0.8.5", "tempfile", "url", @@ -3044,7 +2890,7 @@ version = "47.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7a4e4ce3802609be38eeb607ee72f6fe86c3091460de9dbfae9e18db423b3964" dependencies = [ - "arrow 55.1.0", + "arrow", "chrono", "datafusion-common", "datafusion-doc", @@ -3065,7 +2911,7 @@ version = "47.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "422ac9cf3b22bbbae8cdf8ceb33039107fde1b5492693168f13bd566b1bcc839" dependencies = [ - "arrow 55.1.0", + "arrow", "datafusion-common", "indexmap 2.9.0", "itertools 0.14.0", @@ -3078,8 +2924,8 @@ version = "47.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2ddf0a0a2db5d2918349c978d42d80926c6aa2459cd8a3c533a84ec4bb63479e" dependencies = [ - "arrow 55.1.0", - "arrow-buffer 55.1.0", + "arrow", + "arrow-buffer", "base64 0.22.1", "blake2", "blake3", @@ -3108,7 +2954,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "408a05dafdc70d05a38a29005b8b15e21b0238734dab1e98483fcb58038c5aba" dependencies = [ "ahash 0.8.12", - "arrow 55.1.0", + "arrow", "datafusion-common", "datafusion-doc", "datafusion-execution", @@ -3129,7 +2975,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "756d21da2dd6c9bef97af1504970ff56cbf35d03fbd4ffd62827f02f4d2279d4" dependencies = [ "ahash 0.8.12", - "arrow 55.1.0", + "arrow", "datafusion-common", "datafusion-expr-common", "datafusion-physical-expr-common", @@ -3141,8 +2987,8 @@ version = "47.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8d8d50f6334b378930d992d801a10ac5b3e93b846b39e4a05085742572844537" dependencies = [ - "arrow 55.1.0", - "arrow-ord 55.1.0", + "arrow", + "arrow-ord", "datafusion-common", "datafusion-doc", "datafusion-execution", @@ -3162,13 +3008,13 @@ version = "47.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cc9a97220736c8fff1446e936be90d57216c06f28969f9ffd3b72ac93c958c8a" dependencies = [ - "arrow 55.1.0", + "arrow", "async-trait", "datafusion-catalog", "datafusion-common", "datafusion-expr", "datafusion-physical-plan", - "parking_lot 0.12.3", + "parking_lot 0.12.4", "paste", ] @@ -3216,7 +3062,7 @@ version = "47.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "084d9f979c4b155346d3c34b18f4256e6904ded508e9554d90fed416415c3515" dependencies = [ - "arrow 55.1.0", + "arrow", "chrono", "datafusion-common", "datafusion-expr", @@ -3236,7 +3082,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "64c536062b0076f4e30084065d805f389f9fe38af0ca75bcbac86bc5e9fbab65" dependencies = [ "ahash 0.8.12", - "arrow 55.1.0", + "arrow", "datafusion-common", "datafusion-expr", "datafusion-expr-common", @@ -3258,7 +3104,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8a92b53b3193fac1916a1c5b8e3f4347c526f6822e56b71faa5fb372327a863" dependencies = [ "ahash 0.8.12", - "arrow 55.1.0", + "arrow", "datafusion-common", "datafusion-expr-common", "hashbrown 0.14.5", @@ -3271,7 +3117,7 @@ version = "47.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6fa0a5ac94c7cf3da97bedabd69d6bbca12aef84b9b37e6e9e8c25286511b5e2" dependencies = [ - "arrow 55.1.0", + "arrow", "datafusion-common", "datafusion-execution", "datafusion-expr", @@ -3291,9 +3137,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "690c615db468c2e5fe5085b232d8b1c088299a6c63d87fd960a354a71f7acb55" dependencies = [ "ahash 0.8.12", - "arrow 55.1.0", - "arrow-ord 55.1.0", - "arrow-schema 55.1.0", + "arrow", + "arrow-ord", + "arrow-schema", "async-trait", "chrono", "datafusion-common", @@ -3309,7 +3155,7 @@ dependencies = [ "indexmap 2.9.0", "itertools 0.14.0", "log", - "parking_lot 0.12.3", + "parking_lot 0.12.4", "pin-project-lite", "tokio", ] @@ -3320,7 +3166,7 @@ version = "47.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ad229a134c7406c057ece00c8743c0c34b97f4e72f78b475fe17b66c5e14fa4f" dependencies = [ - "arrow 55.1.0", + "arrow", "async-trait", "dashmap 6.1.0", "datafusion-common", @@ -3334,7 +3180,7 @@ dependencies = [ "itertools 0.14.0", "log", "object_store", - "parking_lot 0.12.3", + "parking_lot 0.12.4", "tokio", ] @@ -3344,7 +3190,7 @@ version = "47.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "64f6ab28b72b664c21a27b22a2ff815fd390ed224c26e89a93b5a8154a4e8607" dependencies = [ - "arrow 55.1.0", + "arrow", "bigdecimal", "datafusion-common", "datafusion-expr", @@ -3385,7 +3231,7 @@ dependencies = [ "once_cell", "percent-encoding", "serde", - "sourcemap 9.2.1", + "sourcemap 9.2.2", "swc_atoms", "swc_common", "swc_config", @@ -3463,7 +3309,7 @@ dependencies = [ "indexmap 2.9.0", "log", "once_cell", - "parking_lot 0.12.3", + "parking_lot 0.12.4", "serde", "serde_json", "sha2 0.10.9", @@ -3545,7 +3391,7 @@ dependencies = [ "indexmap 2.9.0", "libc", "memoffset", - "parking_lot 0.12.3", + "parking_lot 0.12.4", "percent-encoding", "pin-project", "serde", @@ -3670,7 +3516,7 @@ dependencies = [ "http 1.3.1", "http-body-util", "hyper 1.6.0", - "hyper-rustls 0.27.6", + "hyper-rustls 0.27.7", "hyper-util", "ipnet", "percent-encoding", @@ -3790,7 +3636,7 @@ dependencies = [ "log", "once_cell", "os_pipe", - "parking_lot 0.12.3", + "parking_lot 0.12.4", "pin-project", "rand 0.8.5", "tokio", @@ -4170,7 +4016,7 @@ dependencies = [ "log", "node_resolver", "once_cell", - "parking_lot 0.12.3", + "parking_lot 0.12.4", "sys_traits", "thiserror 2.0.12", "url", @@ -4276,7 +4122,7 @@ dependencies = [ "deno_tls", "http-body-util", "hyper 1.6.0", - "hyper-rustls 0.27.6", + "hyper-rustls 0.27.7", "hyper-util", "log", "once_cell", @@ -4322,12 +4168,12 @@ dependencies = [ [[package]] name = "deno_unsync" -version = "0.4.3" +version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "47c618b51088b3ac67f15c69b3ed7620ba3a7d495e5a090186df9424b5ab623e" +checksum = "6742a724e8becb372a74c650a1aefb8924a5b8107f7d75b3848763ea24b27a87" dependencies = [ - "futures", - "parking_lot 0.12.3", + "futures-util", + "parking_lot 0.12.4", "tokio", ] @@ -4806,17 +4652,16 @@ dependencies = [ [[package]] name = "duckdb" -version = "1.2.2" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49ac283b6621e3becf8014d1efa655522794075834c72f744573debef9c9f6c8" +checksum = "379464d1fbb69d1bbda7c11d96fafde6f2bdb78c67e630e6339b048e6e45107e" dependencies = [ - "arrow 54.3.1", + "arrow", "cast", "fallible-iterator 0.3.0", "fallible-streaming-iterator", "hashlink 0.9.1", "libduckdb-sys", - "memchr", "num-integer", "rust_decimal", "smallvec", @@ -5270,9 +5115,9 @@ dependencies = [ [[package]] name = "flate2" -version = "1.1.1" +version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ced92e76e966ca2fd84c8f7aa01a4aea65b0eb6648d72f7c8f3e2764a67fece" +checksum = "4a3d7db9596fecd151c5f638c0ee5d5bd487b6e0ea232e5dc96d5250f6f94b1d" dependencies = [ "crc32fast", "libz-rs-sys", @@ -5483,7 +5328,7 @@ checksum = "1d930c203dd0b6ff06e0201a4a2fe9149b43c684fd4420555b26d21b1a02956f" dependencies = [ "futures-core", "lock_api", - "parking_lot 0.12.3", + "parking_lot 0.12.4", ] [[package]] @@ -5845,11 +5690,11 @@ dependencies = [ [[package]] name = "getopts" -version = "0.2.21" +version = "0.2.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14dbbfd5c71d70241ecf9e6f13737f7b5ce823821063188d7e46c41d371eebd5" +checksum = "cba6ae63eb948698e300f645f87c70f76630d505f23b8907cf1e193ee85048c1" dependencies = [ - "unicode-width 0.1.14", + "unicode-width 0.2.0", ] [[package]] @@ -5990,7 +5835,7 @@ dependencies = [ "google-cloud-token", "home", "jsonwebtoken 9.3.1", - "reqwest 0.12.17", + "reqwest 0.12.19", "serde", "serde_json", "thiserror 1.0.69", @@ -6033,7 +5878,7 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d901aeb453fd80e51d64df4ee005014f6cf39f2d736dd64f7239c132d9d39a6a" dependencies = [ - "reqwest 0.12.17", + "reqwest 0.12.19", "thiserror 1.0.69", "tokio", ] @@ -6105,7 +5950,7 @@ checksum = "b89c83349105e3732062a895becfc71a8f921bb71ecbbdd8ff99263e3b53a0ca" dependencies = [ "bitflags 2.9.1", "gpu-descriptor-types", - "hashbrown 0.15.3", + "hashbrown 0.15.4", ] [[package]] @@ -6220,9 +6065,9 @@ dependencies = [ [[package]] name = "hashbrown" -version = "0.15.3" +version = "0.15.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84b26c544d002229e640969970a2e74021aadf6e2f96372b9c58eff97de08eb3" +checksum = "5971ac85611da7067dbfcabef3c70ebb5606018acd9e2a3903a0da507521e0d5" dependencies = [ "allocator-api2", "equivalent", @@ -6255,7 +6100,7 @@ version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7382cf6263419f2d8df38c55d7da83da5c18aef87fc7a7fc1fb1e344edfe14c1" dependencies = [ - "hashbrown 0.15.3", + "hashbrown 0.15.4", ] [[package]] @@ -6286,9 +6131,9 @@ checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" [[package]] name = "hermit-abi" -version = "0.3.9" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024" +checksum = "f154ce46856750ed433c8649605bf7ed2de3bc35fd9d2a9f30cddd873c80cb08" [[package]] name = "hex" @@ -6357,7 +6202,7 @@ dependencies = [ "ipconfig", "moka", "once_cell", - "parking_lot 0.12.3", + "parking_lot 0.12.4", "rand 0.9.0", "resolv-conf", "serde", @@ -6577,9 +6422,9 @@ dependencies = [ [[package]] name = "hyper-rustls" -version = "0.27.6" +version = "0.27.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03a01595e11bdcec50946522c32dde3fc6914743000a68b93000965f2f02406d" +checksum = "e3c93eb611681b207e1fe55d5a71ecf91572ec8a6705cdb6857f7d8d5242cf58" dependencies = [ "http 1.3.1", "hyper 1.6.0", @@ -6637,9 +6482,9 @@ dependencies = [ [[package]] name = "hyper-util" -version = "0.1.13" +version = "0.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1c293b6b3d21eca78250dc7dbebd6b9210ec5530e038cbfe0661b5c47ab06e8" +checksum = "dc2fdfdbff08affe55bb779f33b053aa1fe5dd5b54c257343c17edfa55711bdb" dependencies = [ "base64 0.22.1", "bytes", @@ -6883,7 +6728,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cea70ddb795996207ad57735b50c5982d8844f38ba9ee5f1aedcfb708a2aa11e" dependencies = [ "equivalent", - "hashbrown 0.15.3", + "hashbrown 0.15.4", "serde", ] @@ -7349,9 +7194,9 @@ checksum = "d750af042f7ef4f724306de029d18836c26c1765a54a6a3f094cbd23a7267ffa" [[package]] name = "libduckdb-sys" -version = "1.2.2" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12cac9d03484c43fefac8b2066a253c9b0b3b0cd02cbe02a9ea2312f7e382618" +checksum = "25d3f1defe457d1ac0fbaef0fe6926953cb33419dd3c8dbac53882d020bee697" dependencies = [ "autocfg", "cc", @@ -7454,9 +7299,9 @@ dependencies = [ [[package]] name = "libz-rs-sys" -version = "0.5.0" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6489ca9bd760fe9642d7644e827b0c9add07df89857b0416ee15c1cc1a3b8c5a" +checksum = "172a788537a2221661b480fee8dc5f96c580eb34fa88764d3205dc356c7e4221" dependencies = [ "zlib-rs", ] @@ -7505,9 +7350,9 @@ checksum = "b4ce301924b7887e9d637144fdade93f9dfff9b60981d4ac161db09720d39aa5" [[package]] name = "lock_api" -version = "0.4.12" +version = "0.4.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07af8b9cdd281b7915f413fa73f29ebd5d55d0d3f0155584dade1ff18cea1b17" +checksum = "96936507f153605bddfcda068dd804796c84324ed2510809e5b2a624c81da765" dependencies = [ "autocfg", "scopeguard", @@ -7548,7 +7393,7 @@ version = "0.12.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "234cf4f4a04dc1f57e24b96cc0cd600cf2af460d4161ac5ecdd0af8e1f3b2a38" dependencies = [ - "hashbrown 0.15.3", + "hashbrown 0.15.4", ] [[package]] @@ -7557,7 +7402,7 @@ version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9f8cc7106155f10bdf99a6f379688f543ad6596a415375b36a59a054ceda1198" dependencies = [ - "hashbrown 0.15.3", + "hashbrown 0.15.4", ] [[package]] @@ -7606,9 +7451,9 @@ dependencies = [ [[package]] name = "macro_rules_attribute" -version = "0.2.0" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a82271f7bc033d84bbca59a3ce3e4159938cb08a9c3aebbe54d215131518a13" +checksum = "65049d7923698040cd0b1ddcced9b0eb14dd22c5f86ae59c3740eab64a676520" dependencies = [ "macro_rules_attribute-proc_macro", "paste", @@ -7616,9 +7461,9 @@ dependencies = [ [[package]] name = "macro_rules_attribute-proc_macro" -version = "0.2.0" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8dd856d451cc0da70e2ef2ce95a18e39a93b7558bedf10201ad28503f918568" +checksum = "670fdfda89751bc4a84ac13eaa63e205cf0fd22b4c9a5fbfa085b63c1f1d3a30" [[package]] name = "magic-crypt" @@ -7975,7 +7820,7 @@ dependencies = [ "crossbeam-epoch", "crossbeam-utils", "loom", - "parking_lot 0.12.3", + "parking_lot 0.12.4", "portable-atomic", "rustc_version 0.4.1", "smallvec", @@ -8084,7 +7929,7 @@ dependencies = [ "tokio", "tokio-native-tls", "tokio-util", - "twox-hash 2.1.0", + "twox-hash 2.1.1", "url", ] @@ -8593,9 +8438,9 @@ dependencies = [ [[package]] name = "num_cpus" -version = "1.16.0" +version = "1.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" +checksum = "91df4bbde75afed763b708b7eee1e8e7651e02d97f6d5dd763e89367e957b23b" dependencies = [ "hermit-abi", "libc", @@ -8639,7 +8484,7 @@ dependencies = [ "getrandom 0.2.16", "http 1.3.1", "rand 0.8.5", - "reqwest 0.12.17", + "reqwest 0.12.19", "serde", "serde_json", "serde_path_to_error", @@ -8684,11 +8529,11 @@ dependencies = [ "hyper 1.6.0", "itertools 0.14.0", "md-5 0.10.6", - "parking_lot 0.12.3", + "parking_lot 0.12.4", "percent-encoding", "quick-xml 0.37.5", "rand 0.9.0", - "reqwest 0.12.17", + "reqwest 0.12.19", "ring 0.17.14", "serde", "serde_json", @@ -8816,9 +8661,9 @@ dependencies = [ [[package]] name = "openssl" -version = "0.10.72" +version = "0.10.73" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fedfea7d58a1f73118430a55da6a286e7b044961736ce96a16a17068ea25e5da" +checksum = "8505734d46c8ab1e19a1dce3aef597ad87dcb4c37e7188231769bd6bd51cebf8" dependencies = [ "bitflags 2.9.1", "cfg-if", @@ -8857,9 +8702,9 @@ dependencies = [ [[package]] name = "openssl-sys" -version = "0.9.108" +version = "0.9.109" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e145e1651e858e820e4860f7b9c5e169bc1d8ce1c86043be79fa7b7634821847" +checksum = "90096e2e47630d78b7d1c20952dc621f957103f8bc2c8359ec81290d75238571" dependencies = [ "cc", "libc", @@ -9122,12 +8967,12 @@ dependencies = [ [[package]] name = "parking_lot" -version = "0.12.3" +version = "0.12.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1bf18183cf54e8d6059647fc3063646a1801cf30896933ec2311622cc4b9a27" +checksum = "70d58bf43669b5795d1576d0641cfb6fbb2057bf629506267a92807158584a13" dependencies = [ "lock_api", - "parking_lot_core 0.9.10", + "parking_lot_core 0.9.11", ] [[package]] @@ -9146,9 +8991,9 @@ dependencies = [ [[package]] name = "parking_lot_core" -version = "0.9.10" +version = "0.9.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8" +checksum = "bc838d2a56b5b1a6c25f55575dfc605fabb63bb2365f6c2353ef9159aa69e4a5" dependencies = [ "cfg-if", "libc", @@ -9164,13 +9009,13 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "be7b2d778f6b841d37083ebdf32e33a524acde1266b5884a8ca29bf00dfa1231" dependencies = [ "ahash 0.8.12", - "arrow-array 55.1.0", - "arrow-buffer 55.1.0", - "arrow-cast 55.1.0", - "arrow-data 55.1.0", + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-data", "arrow-ipc", - "arrow-schema 55.1.0", - "arrow-select 55.1.0", + "arrow-schema", + "arrow-select", "base64 0.22.1", "brotli 8.0.1", "bytes", @@ -9178,7 +9023,7 @@ dependencies = [ "flate2", "futures", "half", - "hashbrown 0.15.3", + "hashbrown 0.15.4", "lz4_flex", "num", "num-bigint", @@ -9189,7 +9034,7 @@ dependencies = [ "snap", "thrift", "tokio", - "twox-hash 2.1.0", + "twox-hash 2.1.1", "zstd", ] @@ -9473,9 +9318,9 @@ dependencies = [ [[package]] name = "portable-atomic" -version = "1.11.0" +version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "350e9b48cbc6b0e028b0473b114454c6316e57336ee184ceab6e53f72c178b3e" +checksum = "f84267b20a16ea918e43c6a88433c2d54fa145c92a811b5b047ccbe153674483" [[package]] name = "postgres-native-tls" @@ -9594,9 +9439,9 @@ checksum = "c6fa0831dd7cc608c38a5e323422a0077678fa5744aa2be4ad91c4ece8eec8d5" [[package]] name = "prettyplease" -version = "0.2.32" +version = "0.2.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "664ec5419c51e34154eec046ebcba56312d5a2fc3b09a06da188e1ad21afadf6" +checksum = "9dee91521343f4c5c6a63edd65e54f31f5c92fe8978c40a4282f8372194c6a7d" dependencies = [ "proc-macro2", "syn 2.0.101", @@ -9617,7 +9462,7 @@ version = "3.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "edce586971a4dfaa28950c6f18ed55e0406c1ab88bbce2c6f6293a7aaba73d35" dependencies = [ - "toml_edit 0.22.26", + "toml_edit 0.22.27", ] [[package]] @@ -9753,7 +9598,7 @@ dependencies = [ "fnv", "lazy_static", "memchr", - "parking_lot 0.12.3", + "parking_lot 0.12.4", "thiserror 2.0.12", ] @@ -9920,8 +9765,8 @@ checksum = "6b450dad8382b1b95061d5ca1eb792081fb082adf48c678791fe917509596d5f" dependencies = [ "ahash 0.8.12", "equivalent", - "hashbrown 0.15.3", - "parking_lot 0.12.3", + "hashbrown 0.15.4", + "parking_lot 0.12.4", ] [[package]] @@ -10376,9 +10221,9 @@ dependencies = [ [[package]] name = "reqwest" -version = "0.12.17" +version = "0.12.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3f27698fc5799d2a281528a908bd874973953ca2e7bc2311637e10c263c723b" +checksum = "a2f8e5513d63f2e5b386eb5106dc67eaf3f84e95258e210489136b8b92ad6119" dependencies = [ "async-compression", "base64 0.22.1", @@ -10391,7 +10236,7 @@ dependencies = [ "http-body 1.0.1", "http-body-util", "hyper 1.6.0", - "hyper-rustls 0.27.6", + "hyper-rustls 0.27.7", "hyper-tls 0.6.0", "hyper-util", "ipnet", @@ -10434,7 +10279,7 @@ dependencies = [ "anyhow", "async-trait", "http 1.3.1", - "reqwest 0.12.17", + "reqwest 0.12.19", "serde", "thiserror 1.0.69", "tower-service", @@ -10453,7 +10298,7 @@ dependencies = [ "http 1.3.1", "hyper 1.6.0", "parking_lot 0.11.2", - "reqwest 0.12.17", + "reqwest 0.12.19", "reqwest-middleware", "retry-policies", "thiserror 1.0.69", @@ -11385,9 +11230,9 @@ dependencies = [ [[package]] name = "serde_spanned" -version = "0.6.8" +version = "0.6.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87607cb1398ed59d48732e575a4c28a7a8ebf2454b964fe3f224f2afc07909e1" +checksum = "bf41e0cfaf7226dca15e8197172c295a782857fcb97fad1808a166870dee75a3" dependencies = [ "serde", ] @@ -11672,9 +11517,9 @@ dependencies = [ [[package]] name = "smallvec" -version = "1.15.0" +version = "1.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8917285742e9f3e1683f0a9c4e6b57960b7314d0b08d30d1ecd426713ee2eee9" +checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" dependencies = [ "serde", ] @@ -11733,9 +11578,9 @@ dependencies = [ [[package]] name = "sourcemap" -version = "9.2.1" +version = "9.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bdee719193ae5c919a3ee43f64c2c0dd87f9b9a451d67918a2a5ec2e3c70561c" +checksum = "e22afbcb92ce02d23815b9795523c005cb9d3c214f8b7a66318541c240ea7935" dependencies = [ "base64-simd 0.8.0", "bitvec", @@ -11864,7 +11709,7 @@ dependencies = [ "futures-intrusive", "futures-io", "futures-util", - "hashbrown 0.15.3", + "hashbrown 0.15.4", "hashlink 0.10.0", "indexmap 2.9.0", "log", @@ -12237,7 +12082,7 @@ dependencies = [ "rustc-hash 1.1.0", "serde", "siphasher 0.3.11", - "sourcemap 9.2.1", + "sourcemap 9.2.2", "swc_allocator", "swc_atoms", "swc_eq_ignore_macros", @@ -12301,7 +12146,7 @@ dependencies = [ "num-bigint", "once_cell", "serde", - "sourcemap 9.2.1", + "sourcemap 9.2.2", "swc_allocator", "swc_atoms", "swc_common", @@ -13195,7 +13040,7 @@ dependencies = [ "bytes", "libc", "mio 1.0.4", - "parking_lot 0.12.3", + "parking_lot 0.12.4", "pin-project-lite", "signal-hook-registry", "socket2", @@ -13259,7 +13104,7 @@ dependencies = [ "futures-channel", "futures-util", "log", - "parking_lot 0.12.3", + "parking_lot 0.12.4", "percent-encoding", "phf", "pin-project-lite", @@ -13285,7 +13130,7 @@ dependencies = [ "futures-channel", "futures-util", "log", - "parking_lot 0.12.3", + "parking_lot 0.12.4", "percent-encoding", "phf", "pin-project-lite", @@ -13402,7 +13247,7 @@ dependencies = [ "futures-io", "futures-sink", "futures-util", - "hashbrown 0.15.3", + "hashbrown 0.15.4", "pin-project-lite", "slab", "tokio", @@ -13443,9 +13288,9 @@ dependencies = [ [[package]] name = "toml_datetime" -version = "0.6.9" +version = "0.6.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3da5db5a963e24bc68be8b17b6fa82814bb22ee8660f192bb182771d498f09a3" +checksum = "22cddaf88f4fbc13c51aebbf5f8eceb5c7c5a9da2ac40a13519eb5b0a0e8f11c" dependencies = [ "serde", ] @@ -13465,9 +13310,9 @@ dependencies = [ [[package]] name = "toml_edit" -version = "0.22.26" +version = "0.22.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "310068873db2c5b3e7659d2cc35d21855dbafa50d1ce336397c666e3cb08137e" +checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a" dependencies = [ "indexmap 2.9.0", "toml_datetime", @@ -13556,7 +13401,7 @@ dependencies = [ "cookie 0.18.1", "futures-util", "http 1.3.1", - "parking_lot 0.12.3", + "parking_lot 0.12.4", "pin-project-lite", "tower-layer", "tower-service", @@ -13564,9 +13409,9 @@ dependencies = [ [[package]] name = "tower-http" -version = "0.6.4" +version = "0.6.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fdb0c213ca27a9f57ab69ddb290fd80d970922355b83ae380b395d3986b8a2e" +checksum = "adc82fd73de2a9722ac5da747f12383d2bfdb93591ee6c58486e0097890f05f2" dependencies = [ "async-compression", "bitflags 2.9.1", @@ -13624,9 +13469,9 @@ dependencies = [ [[package]] name = "tracing-attributes" -version = "0.1.28" +version = "0.1.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "395ae124c09f9e6918a2310af6038fba074bcf474ac352496d5910dd59a2226d" +checksum = "1b1ffbcf9c6f6b99d386e7444eb608ba646ae452a36b39737deb9663b610f662" dependencies = [ "proc-macro2", "quote", @@ -13635,9 +13480,9 @@ dependencies = [ [[package]] name = "tracing-core" -version = "0.1.33" +version = "0.1.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e672c95779cf947c5311f83787af4fa8fffd12fb27e4993211a84bdfd9610f9c" +checksum = "b9d12581f227e93f094d3af2ae690a574abb8a2b9b7a96e7cfe9647b2b617678" dependencies = [ "once_cell", "valuable", @@ -13671,7 +13516,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ba3beec919fbdf99d719de8eda6adae3281f8a5b71ae40431f44dc7423053d34" dependencies = [ "loki-api", - "reqwest 0.12.17", + "reqwest 0.12.19", "serde", "serde_json", "snap", @@ -13831,9 +13676,9 @@ dependencies = [ [[package]] name = "twox-hash" -version = "2.1.0" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7b17f197b3050ba473acf9181f7b1d3b66d1cf7356c6cc57886662276e65908" +checksum = "8b907da542cbced5261bd3256de1b3a1bf340a3d37f93425a07362a1d687de56" [[package]] name = "typed-arena" @@ -14526,7 +14371,7 @@ dependencies = [ "log", "naga", "once_cell", - "parking_lot 0.12.3", + "parking_lot 0.12.4", "profiling", "raw-window-handle", "ron", @@ -14568,7 +14413,7 @@ dependencies = [ "ndk-sys", "objc", "once_cell", - "parking_lot 0.12.3", + "parking_lot 0.12.4", "profiling", "range-alloc", "raw-window-handle", @@ -14667,7 +14512,7 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "windmill" -version = "1.493.2" +version = "1.496.3" dependencies = [ "anyhow", "axum", @@ -14687,7 +14532,7 @@ dependencies = [ "prometheus", "quote", "rand 0.9.0", - "reqwest 0.12.17", + "reqwest 0.12.19", "rustls 0.23.27", "serde", "serde_json", @@ -14718,7 +14563,7 @@ dependencies = [ [[package]] name = "windmill-api" -version = "1.493.2" +version = "1.496.3" dependencies = [ "anyhow", "argon2", @@ -14777,7 +14622,7 @@ dependencies = [ "rand 0.9.0", "rdkafka", "regex", - "reqwest 0.12.17", + "reqwest 0.12.19", "rmcp", "rsa", "rumqttc", @@ -14827,7 +14672,7 @@ dependencies = [ [[package]] name = "windmill-api-client" -version = "1.493.2" +version = "1.496.3" dependencies = [ "base64 0.22.1", "chrono", @@ -14842,7 +14687,7 @@ dependencies = [ [[package]] name = "windmill-audit" -version = "1.493.2" +version = "1.496.3" dependencies = [ "chrono", "serde", @@ -14855,7 +14700,7 @@ dependencies = [ [[package]] name = "windmill-autoscaling" -version = "1.493.2" +version = "1.496.3" dependencies = [ "anyhow", "serde", @@ -14869,7 +14714,7 @@ dependencies = [ [[package]] name = "windmill-common" -version = "1.493.2" +version = "1.496.3" dependencies = [ "anyhow", "async-stream", @@ -14911,7 +14756,7 @@ dependencies = [ "quick_cache", "rand 0.9.0", "regex", - "reqwest 0.12.17", + "reqwest 0.12.19", "reqwest-middleware", "reqwest-retry", "semver 1.0.26", @@ -14945,7 +14790,7 @@ dependencies = [ [[package]] name = "windmill-git-sync" -version = "1.493.2" +version = "1.496.3" dependencies = [ "regex", "serde", @@ -14959,7 +14804,7 @@ dependencies = [ [[package]] name = "windmill-indexer" -version = "1.493.2" +version = "1.496.3" dependencies = [ "anyhow", "bytes", @@ -14982,7 +14827,7 @@ dependencies = [ [[package]] name = "windmill-macros" -version = "1.493.2" +version = "1.496.3" dependencies = [ "itertools 0.14.0", "lazy_static", @@ -14994,7 +14839,7 @@ dependencies = [ [[package]] name = "windmill-parser" -version = "1.493.2" +version = "1.496.3" dependencies = [ "convert_case 0.6.0", "serde", @@ -15003,7 +14848,7 @@ dependencies = [ [[package]] name = "windmill-parser-bash" -version = "1.493.2" +version = "1.496.3" dependencies = [ "anyhow", "lazy_static", @@ -15015,7 +14860,7 @@ dependencies = [ [[package]] name = "windmill-parser-csharp" -version = "1.493.2" +version = "1.496.3" dependencies = [ "anyhow", "serde_json", @@ -15027,7 +14872,7 @@ dependencies = [ [[package]] name = "windmill-parser-go" -version = "1.493.2" +version = "1.496.3" dependencies = [ "anyhow", "gosyn", @@ -15039,7 +14884,7 @@ dependencies = [ [[package]] name = "windmill-parser-graphql" -version = "1.493.2" +version = "1.496.3" dependencies = [ "anyhow", "lazy_static", @@ -15051,7 +14896,7 @@ dependencies = [ [[package]] name = "windmill-parser-java" -version = "1.493.2" +version = "1.496.3" dependencies = [ "anyhow", "serde_json", @@ -15063,7 +14908,7 @@ dependencies = [ [[package]] name = "windmill-parser-nu" -version = "1.493.2" +version = "1.496.3" dependencies = [ "anyhow", "nu-parser", @@ -15074,7 +14919,7 @@ dependencies = [ [[package]] name = "windmill-parser-php" -version = "1.493.2" +version = "1.496.3" dependencies = [ "anyhow", "itertools 0.14.0", @@ -15085,7 +14930,7 @@ dependencies = [ [[package]] name = "windmill-parser-py" -version = "1.493.2" +version = "1.496.3" dependencies = [ "anyhow", "itertools 0.14.0", @@ -15096,7 +14941,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-imports" -version = "1.493.2" +version = "1.496.3" dependencies = [ "anyhow", "async-recursion", @@ -15119,7 +14964,7 @@ dependencies = [ [[package]] name = "windmill-parser-rust" -version = "1.493.2" +version = "1.496.3" dependencies = [ "anyhow", "convert_case 0.6.0", @@ -15136,7 +14981,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql" -version = "1.493.2" +version = "1.496.3" dependencies = [ "anyhow", "lazy_static", @@ -15148,7 +14993,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts" -version = "1.493.2" +version = "1.496.3" dependencies = [ "anyhow", "lazy_static", @@ -15166,7 +15011,7 @@ dependencies = [ [[package]] name = "windmill-parser-wasm" -version = "1.493.2" +version = "1.496.3" dependencies = [ "anyhow", "getrandom 0.2.16", @@ -15190,7 +15035,7 @@ dependencies = [ [[package]] name = "windmill-parser-yaml" -version = "1.493.2" +version = "1.496.3" dependencies = [ "anyhow", "serde_json", @@ -15200,7 +15045,7 @@ dependencies = [ [[package]] name = "windmill-queue" -version = "1.493.2" +version = "1.496.3" dependencies = [ "anyhow", "async-recursion", @@ -15217,7 +15062,7 @@ dependencies = [ "lazy_static", "prometheus", "regex", - "reqwest 0.12.17", + "reqwest 0.12.19", "serde", "serde_json", "serde_urlencoded", @@ -15233,7 +15078,7 @@ dependencies = [ [[package]] name = "windmill-sql-datatype-parser-wasm" -version = "1.493.2" +version = "1.496.3" dependencies = [ "wasm-bindgen", "wasm-bindgen-test", @@ -15243,7 +15088,7 @@ dependencies = [ [[package]] name = "windmill-worker" -version = "1.493.2" +version = "1.496.3" dependencies = [ "anyhow", "async-recursion", @@ -15295,7 +15140,7 @@ dependencies = [ "prometheus", "rand 0.9.0", "regex", - "reqwest 0.12.17", + "reqwest 0.12.19", "reqwest-middleware", "rust_decimal", "serde", @@ -15411,7 +15256,7 @@ dependencies = [ "windows-interface 0.59.1", "windows-link", "windows-result 0.3.4", - "windows-strings 0.4.2", + "windows-strings", ] [[package]] @@ -15509,13 +15354,13 @@ dependencies = [ [[package]] name = "windows-registry" -version = "0.4.0" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4286ad90ddb45071efd1a66dfa43eb02dd0dfbae1545ad6cc3c51cf34d7e8ba3" +checksum = "b3bab093bdd303a1240bb99b8aba8ea8a69ee19d34c9e2ef9594e708a4878820" dependencies = [ + "windows-link", "windows-result 0.3.4", - "windows-strings 0.3.1", - "windows-targets 0.53.0", + "windows-strings", ] [[package]] @@ -15536,15 +15381,6 @@ dependencies = [ "windows-link", ] -[[package]] -name = "windows-strings" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87fa48cc5d406560701792be122a10132491cff9d0aeb23583cc2dcafc847319" -dependencies = [ - "windows-link", -] - [[package]] name = "windows-strings" version = "0.4.2" @@ -16082,9 +15918,9 @@ dependencies = [ [[package]] name = "zlib-rs" -version = "0.5.0" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "868b928d7949e09af2f6086dfc1e01936064cc7a819253bce650d4e2a2d63ba8" +checksum = "626bd9fa9734751fc50d6060752170984d7053f5a39061f524cda68023d4db8a" [[package]] name = "zstd" diff --git a/backend/Cargo.toml b/backend/Cargo.toml index ea651b94e0..84cb427c69 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "windmill" -version = "1.493.2" +version = "1.496.3" authors.workspace = true edition.workspace = true @@ -32,7 +32,7 @@ members = [ ] [workspace.package] -version = "1.493.2" +version = "1.496.3" authors = ["Ruben Fiszel "] edition = "2021" @@ -49,6 +49,7 @@ lto = "thin" [features] default = [] +private = ["windmill-api/private", "windmill-autoscaling/private", "windmill-common/private", "windmill-git-sync/private", "windmill-indexer/private", "windmill-queue/private", "windmill-worker/private"] agent_worker_server = ["windmill-api/agent_worker_server"] enterprise = ["windmill-worker/enterprise", "windmill-queue/enterprise", "windmill-api/enterprise", "dep:windmill-autoscaling", "windmill-autoscaling/enterprise", "windmill-git-sync/enterprise", "windmill-common/prometheus", "windmill-common/enterprise"] enterprise_saml = ["windmill-api/enterprise_saml", "oauth2"] @@ -94,8 +95,10 @@ php = ["windmill-worker/php"] csharp = ["windmill-worker/csharp"] nu = ["windmill-worker/nu"] java = ["windmill-worker/java"] -all_languages = [ "python", "deno_core", "rust", "mysql", "oracledb", "duckdb", "mssql", "bigquery", "csharp", "nu", "php", "java"] - +all_languages = ["python", "deno_core", "rust", "mysql", "oracledb", "duckdb", "mssql", "bigquery", "csharp", "nu", "php", "java"] +# For windows we have another set of languages enabled +# NOTE: DuckDB is ignored because of compilation problems +all_languages_windows = ["python", "deno_core", "rust", "mysql", "oracledb", "mssql", "bigquery", "csharp", "nu", "php", "java"] [patch.crates-io] object_store = { git = "https://github.com/apache/arrow-rs-object-store", rev = "36752c975d4f29e20b57c91f81a10872dcd48ae7" } diff --git a/backend/all_features_oss.sh b/backend/all_features_oss.sh new file mode 100755 index 0000000000..5871bc656f --- /dev/null +++ b/backend/all_features_oss.sh @@ -0,0 +1,20 @@ +# This script outputs all features except private. Usage : +# > cargo build --features $(./all_features_oss.sh) + +#!/bin/bash + +# Path to the Cargo.toml file +CARGO_TOML_PATH="./Cargo.toml" + +# Extract features from Cargo.toml and output them separated by commas +if [[ -f "$CARGO_TOML_PATH" ]]; then + grep -A 100 '\[features\]' "$CARGO_TOML_PATH" | \ + sed -n '/\[features\]/,/^\[/p' | \ + grep -E '^[a-zA-Z0-9_-]+' | \ + grep -v 'private' | \ + cut -d' ' -f1 | \ + paste -sd ',' - +else + echo "Cargo.toml not found at $CARGO_TOML_PATH" + exit 1 +fi \ No newline at end of file diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index be1e897bcd..fe8033ab6e 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -32039f675060b5996951708368bdefe14278d5cd \ No newline at end of file +ab4f856708a056973faea19f10eda5efff546176 diff --git a/backend/migrations/20250604181226_auth_cache_invalidation.down.sql b/backend/migrations/20250604181226_auth_cache_invalidation.down.sql new file mode 100644 index 0000000000..b7f0f12204 --- /dev/null +++ b/backend/migrations/20250604181226_auth_cache_invalidation.down.sql @@ -0,0 +1,4 @@ +-- Remove token invalidation notification trigger + +DROP TRIGGER IF EXISTS token_invalidation_trigger ON token; +DROP FUNCTION IF EXISTS notify_token_invalidation(); diff --git a/backend/migrations/20250604181226_auth_cache_invalidation.up.sql b/backend/migrations/20250604181226_auth_cache_invalidation.up.sql new file mode 100644 index 0000000000..11be3961c0 --- /dev/null +++ b/backend/migrations/20250604181226_auth_cache_invalidation.up.sql @@ -0,0 +1,17 @@ +-- Add token invalidation notification trigger + +CREATE OR REPLACE FUNCTION notify_token_invalidation() +RETURNS TRIGGER AS $$ +BEGIN + -- Only notify for session token deletions when the invalidation settings are enabled + IF OLD.label = 'session' AND OLD.email IS NOT NULL THEN + PERFORM pg_notify('notify_token_invalidation', OLD.token); + END IF; + RETURN OLD; +END; +$$ LANGUAGE plpgsql; + +CREATE TRIGGER token_invalidation_trigger +AFTER DELETE ON token +FOR EACH ROW +EXECUTE FUNCTION notify_token_invalidation(); diff --git a/backend/src/ee.rs b/backend/src/ee_oss.rs similarity index 53% rename from backend/src/ee.rs rename to backend/src/ee_oss.rs index 91816cd1ba..4791a243a3 100644 --- a/backend/src/ee.rs +++ b/backend/src/ee_oss.rs @@ -1,8 +1,13 @@ +#[cfg(feature = "private")] +#[allow(unused)] +pub use crate::ee::*; + +#[cfg(not(feature = "private"))] pub async fn set_license_key(_license_key: String) -> () { // Implementation is not open source } -#[cfg(feature = "enterprise")] +#[cfg(all(feature = "enterprise", not(feature = "private")))] pub async fn verify_license_key() -> () { // Implementation is not open source } diff --git a/backend/src/main.rs b/backend/src/main.rs index a4148d93c6..e469123a53 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -28,7 +28,9 @@ use uuid::Uuid; use windmill_api::HTTP_CLIENT; #[cfg(feature = "enterprise")] -use windmill_common::ee::{maybe_renew_license_key_on_start, LICENSE_KEY_ID, LICENSE_KEY_VALID}; +use windmill_common::ee_oss::{ + maybe_renew_license_key_on_start, LICENSE_KEY_ID, LICENSE_KEY_VALID, +}; use windmill_common::{ agent_workers::build_agent_http_client, @@ -49,9 +51,12 @@ use windmill_common::{ TIMEOUT_WAIT_RESULT_SETTING, }, scripts::ScriptLang, - stats_ee::schedule_stats, + stats_oss::schedule_stats, triggers::TriggerKind, - utils::{hostname, rd_string, Mode, GIT_VERSION, MODE_AND_ADDONS}, + utils::{ + create_default_worker_suffix, create_ssh_agent_worker_suffix, worker_name_with_suffix, + Mode, GIT_VERSION, HOSTNAME, MODE_AND_ADDONS, + }, worker::{ reload_custom_tags_setting, Connection, HUB_CACHE_DIR, TMP_DIR, TMP_LOGS_DIR, WORKER_GROUP, }, @@ -75,8 +80,7 @@ use windmill_worker::{ get_hub_script_content_and_requirements, BUN_BUNDLE_CACHE_DIR, BUN_CACHE_DIR, CSHARP_CACHE_DIR, DENO_CACHE_DIR, DENO_CACHE_DIR_DEPS, DENO_CACHE_DIR_NPM, GO_BIN_CACHE_DIR, GO_CACHE_DIR, JAVA_CACHE_DIR, NU_CACHE_DIR, POWERSHELL_CACHE_DIR, PY310_CACHE_DIR, PY311_CACHE_DIR, - PY312_CACHE_DIR, PY313_CACHE_DIR, RUST_CACHE_DIR, TAR_JAVA_CACHE_DIR, TAR_PY310_CACHE_DIR, - TAR_PY311_CACHE_DIR, TAR_PY312_CACHE_DIR, TAR_PY313_CACHE_DIR, UV_CACHE_DIR, + PY312_CACHE_DIR, PY313_CACHE_DIR, RUST_CACHE_DIR, TAR_JAVA_CACHE_DIR, UV_CACHE_DIR, }; use crate::monitor::{ @@ -98,7 +102,9 @@ const DEFAULT_NUM_WORKERS: usize = 1; const DEFAULT_PORT: u16 = 8000; const DEFAULT_SERVER_BIND_ADDR: Ipv4Addr = Ipv4Addr::new(0, 0, 0, 0); -mod ee; +#[cfg(feature = "private")] +pub mod ee; +mod ee_oss; mod monitor; pub fn setup_deno_runtime() -> anyhow::Result<()> { @@ -261,7 +267,7 @@ async fn windmill_main() -> anyhow::Result<()> { tracing::error!("Failed to install rustls crypto provider"); } - let hostname = hostname(); + let hostname = HOSTNAME.to_owned(); let mode_and_addons = MODE_AND_ADDONS.clone(); let mode = mode_and_addons.mode; @@ -340,7 +346,7 @@ async fn windmill_main() -> anyhow::Result<()> { "Creating http client for cluster using base internal url {}", std::env::var("BASE_INTERNAL_URL").unwrap_or_default() ); - let suffix = windmill_common::utils::worker_suffix(&hostname, &rd_string(5)); + let suffix = create_ssh_agent_worker_suffix(&hostname); ( Connection::Http(build_agent_http_client(&suffix)), Some(suffix), @@ -552,7 +558,7 @@ Windmill Community Edition {GIT_VERSION} _ = indexer_rx.recv() => { tracing::info!("Received killpill, aborting index initialization"); }, - res = windmill_indexer::completed_runs_ee::init_index(&db) => { + res = windmill_indexer::completed_runs_oss::init_index(&db) => { let res = res?; reader = Some(res.0); writer = Some(res.1); @@ -574,7 +580,7 @@ Windmill Community Edition {GIT_VERSION} async { if let Some(db) = conn.as_sql() { if let Some(index_writer) = index_writer2 { - windmill_indexer::completed_runs_ee::run_indexer( + windmill_indexer::completed_runs_oss::run_indexer( db.clone(), index_writer, indexer_rx, @@ -596,7 +602,7 @@ Windmill Community Edition {GIT_VERSION} _ = indexer_rx.recv() => { tracing::info!("Received killpill, aborting index initialization"); }, - res = windmill_indexer::service_logs_ee::init_index(&db, killpill_tx.clone()) => { + res = windmill_indexer::service_logs_oss::init_index(&db, killpill_tx.clone()) => { let res = res?; reader = Some(res.0); writer = Some(res.1); @@ -618,7 +624,7 @@ Windmill Community Edition {GIT_VERSION} async { if let Some(db) = conn.as_sql() { if let Some(log_index_writer) = log_index_writer2 { - windmill_indexer::service_logs_ee::run_indexer( + windmill_indexer::service_logs_oss::run_indexer( db.clone(), log_index_writer, log_indexer_rx, @@ -675,19 +681,21 @@ Windmill Community Edition {GIT_VERSION} let base_internal_url = base_internal_rx.await?; if worker_mode { let mut workers = vec![]; + for i in 0..num_workers { - let suffix: String = if i == 0 && first_suffix.as_ref().is_some() { + let suffix = if i == 0 && first_suffix.is_some() { first_suffix.as_ref().unwrap().clone() } else { - windmill_common::utils::worker_suffix(&hostname, &rd_string(5)) + create_default_worker_suffix(&hostname) }; + let worker_conn = WorkerConn { conn: if i == 0 || mode != Mode::Agent { conn.clone() } else { Connection::Http(build_agent_http_client(&suffix)) }, - worker_name: windmill_common::utils::worker_name_with_suffix( + worker_name: worker_name_with_suffix( mode == Mode::Agent, WORKER_GROUP.as_str(), &suffix, @@ -852,6 +860,11 @@ Windmill Community Edition {GIT_VERSION} } }; }, + "notify_token_invalidation" => { + let token = n.payload(); + tracing::info!("Token invalidation detected for token: {}...", &token[..token.len().min(8)]); + windmill_api::auth::invalidate_token_from_cache(token); + }, "notify_global_setting_change" => { tracing::info!("Global setting change detected: {}", n.payload()); match n.payload() { @@ -884,7 +897,7 @@ Windmill Community Edition {GIT_VERSION} if let Err(e) = load_tag_per_workspace_workspaces(&db).await { tracing::error!("Error loading default tag per workspace workspaces: {e:#}"); } - } + }, SMTP_SETTING => { reload_smtp_config(&db).await; }, @@ -1001,7 +1014,6 @@ Windmill Community Edition {GIT_VERSION} tracing::error!(error = %e, "Could not reload critical alert UI setting"); } }, - a @_ => { tracing::info!("Unrecognized Global Setting Change Payload: {:?}", a); } @@ -1086,7 +1098,7 @@ Windmill Community Edition {GIT_VERSION} tracing::info!("Reloading config after 12 hours"); initial_load(&conn, tx.clone(), worker_mode, server_mode, #[cfg(feature = "parquet")] disable_s3_store).await; #[cfg(feature = "enterprise")] - ee::verify_license_key().await; + ee_oss::verify_license_key().await; } } }, @@ -1174,6 +1186,7 @@ async fn listen_pg(url: &str) -> Option { "notify_webhook_change", "notify_workspace_envs_change", "notify_runnable_version_change", + "notify_token_invalidation", ]; #[cfg(feature = "http_trigger")] @@ -1267,10 +1280,6 @@ pub async fn run_workers( PY311_CACHE_DIR, PY312_CACHE_DIR, PY313_CACHE_DIR, - TAR_PY310_CACHE_DIR, - TAR_PY311_CACHE_DIR, - TAR_PY312_CACHE_DIR, - TAR_PY313_CACHE_DIR, BUN_BUNDLE_CACHE_DIR, GO_CACHE_DIR, GO_BIN_CACHE_DIR, diff --git a/backend/src/monitor.rs b/backend/src/monitor.rs index d2a5657c6c..7b1cfc6d5f 100644 --- a/backend/src/monitor.rs +++ b/backend/src/monitor.rs @@ -29,9 +29,9 @@ use windmill_api::{ }; #[cfg(feature = "enterprise")] -use windmill_common::ee::low_disk_alerts; +use windmill_common::ee_oss::low_disk_alerts; #[cfg(feature = "enterprise")] -use windmill_common::ee::{jobs_waiting_alerts, worker_groups_alerts}; +use windmill_common::ee_oss::{jobs_waiting_alerts, worker_groups_alerts}; use windmill_common::client::AuthedClient; #[cfg(feature = "oauth2")] @@ -41,7 +41,7 @@ use windmill_common::s3_helpers::reload_object_store_setting; use windmill_common::{ agent_workers::DECODED_AGENT_TOKEN, auth::create_token_for_owner, - ee::CriticalErrorChannel, + ee_oss::CriticalErrorChannel, error, flow_status::{FlowStatus, FlowStatusModule}, global_settings::{ @@ -87,9 +87,9 @@ use windmill_worker::{ use windmill_common::s3_helpers::ObjectStoreReload; #[cfg(feature = "enterprise")] -use crate::ee::verify_license_key; +use crate::ee_oss::verify_license_key; -use crate::ee::set_license_key; +use crate::ee_oss::set_license_key; #[cfg(feature = "prometheus")] lazy_static::lazy_static! { @@ -1609,7 +1609,7 @@ pub async fn reload_base_url_setting(conn: &Connection) -> error::Result<()> { if let Some(q) = q_oauth { if let Ok(v) = serde_json::from_value::< - Option>, + Option>, >(q.clone()) { v @@ -1630,7 +1630,7 @@ pub async fn reload_base_url_setting(conn: &Connection) -> error::Result<()> { { if let Some(db) = conn.as_sql() { let mut l = windmill_api::OAUTH_CLIENTS.write().await; - *l = windmill_api::oauth2_ee::build_oauth_clients(&base_url, oauths, db).await + *l = windmill_api::oauth2_oss::build_oauth_clients(&base_url, oauths, db).await .map_err(|e| tracing::error!("Error building oauth clients (is the oauth.json mounted and in correct format? Use '{}' as minimal oauth.json): {}", "{}", e)) .unwrap(); } @@ -1914,12 +1914,12 @@ async fn handle_zombie_jobs(db: &Pool, base_internal_url: &str, worker .await .expect("could not create job token"); - let client = AuthedClient { - base_internal_url: base_internal_url.to_string(), + let client = AuthedClient::new( + base_internal_url.to_string(), + job.workspace_id.to_string(), token, - workspace: job.workspace_id.to_string(), - force_client: None, - }; + None, + ); let last_ping = job.last_ping.clone(); let error_message = format!( @@ -1938,7 +1938,7 @@ async fn handle_zombie_jobs(db: &Pool, base_internal_url: &str, worker None, error::Error::ExecutionErr(error_message), true, - same_worker_tx_never_used, + Some(&same_worker_tx_never_used), "", worker_name, send_result_never_used, diff --git a/backend/substitute_ee_code.sh b/backend/substitute_ee_code.sh index 28b890990b..a1819c75b2 100755 --- a/backend/substitute_ee_code.sh +++ b/backend/substitute_ee_code.sh @@ -4,8 +4,8 @@ script_dirpath="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" root_dirpath="$(cd "${script_dirpath}/.." && pwd)" REVERT="NO" -REVERT_PREVIOUS="NO" COPY="NO" +MOVE_NEW_FILES="NO" EE_CODE_DIR="../windmill-ee-private/" while [[ $# -gt 0 ]]; do @@ -16,13 +16,7 @@ while [[ $# -gt 0 ]]; do # this to work (commit hooks should prevent this from happening, as well as the fact # that we're using symlinks by default). REVERT="YES" - shift - ;; - --revert-previous) - # This is a special case of --revert that will revert to the previous commit. - REVERT="YES" - REVERT_PREVIOUS="YES" - echo "Reverting to previous commit" + MOVE_NEW_FILES="YES" shift ;; -c|--copy) @@ -33,6 +27,11 @@ while [[ $# -gt 0 ]]; do COPY="YES" shift # past argument ;; + -m|--move-new-files) + # This moves all new EE files from the public repository to the private repository. + MOVE_NEW_FILES="YES" + shift # past argument + ;; -d|--dir) # Path to the local directory of the windmill-ee-private repository. By defaults, it # assumes it is cloned next to the Windmill OSS repo. @@ -70,29 +69,34 @@ if [ "$REVERT" == "YES" ]; then for ee_file in $(find ${EE_CODE_DIR} -name "*ee.rs"); do ce_file="${ee_file/${EE_CODE_DIR}/}" ce_file="${root_dirpath}/backend/${ce_file}" - if [ "$REVERT_PREVIOUS" == "YES" ]; then - git checkout HEAD@{3} ${ce_file} || true - else - git restore --staged ${ce_file} || true - git restore ${ce_file} || true - fi + rm ${ce_file} || true done -else +elif [ "$MOVE_NEW_FILES" == "NO" ]; then # This replaces all files in current repo with alternative EE files in windmill-ee-private for ee_file in $(find "${EE_CODE_DIR}" -name "*ee.rs"); do - ce_file="${ee_file/${EE_CODE_DIR}/}" - ce_file="${root_dirpath}/backend/${ce_file}" - if [[ -f "${ce_file}" ]]; then - rm "${ce_file}" - if [ "$COPY" == "YES" ]; then - cp "${ee_file}" "${ce_file}" - echo "File copied '${ee_file}' -->> '${ce_file}'" - else - ln -s "${ee_file}" "${ce_file}" - echo "Symlink created '${ee_file}' -->> '${ce_file}'" - fi + ce_file="${ee_file/${EE_CODE_DIR}/}" + ce_file="${root_dirpath}/backend/${ce_file}" + if [ "$COPY" == "YES" ]; then + cp "${ee_file}" "${ce_file}" + echo "File copied '${ee_file}' -->> '${ce_file}'" else - echo "File ${ce_file} is not a file, ignoring" + ln -s "${ee_file}" "${ce_file}" || true + echo "Symlink created '${ee_file}' -->> '${ce_file}'" fi done fi + +if [ "$MOVE_NEW_FILES" == "YES" ]; then + for ce_file in $(find "${root_dirpath}"/backend/windmill-*/src/ -name "*ee.rs"); do + backend_dirpath="${root_dirpath}/backend/" + ee_file="${ce_file/${backend_dirpath}/}" + ee_file="${EE_CODE_DIR}${ee_file}" + if [ ! -f "${ee_file}" ]; then + mv "${ce_file}" "${ee_file}" + if [ ! "$REVERT" == "YES" ]; then + ln -s "${ee_file}" "${ce_file}" + fi + echo "File moved '${ce_file}' -->> '${ee_file}'" + fi + done +fi \ No newline at end of file diff --git a/backend/tests/worker.rs b/backend/tests/worker.rs index c8299796f8..8792b2eb1d 100644 --- a/backend/tests/worker.rs +++ b/backend/tests/worker.rs @@ -3969,8 +3969,7 @@ async fn assert_lockfile( #[cfg(feature = "python")] #[sqlx::test(fixtures("base", "lockfile_python"))] async fn test_requirements_python(db: Pool) { - let content = r#" -# py: 3.11.11 + let content = r#"# py: ==3.11.11 # requirements: # tiny==0.1.3 @@ -3997,8 +3996,7 @@ def main(): #[sqlx::test(fixtures("base", "lockfile_python"))] async fn test_extra_requirements_python(db: Pool) { { - let content = r#" -# py: ==3.11.11 + let content = r#"# py: ==3.11.11 # extra_requirements: # tiny @@ -4025,8 +4023,7 @@ def main(): #[cfg(feature = "python")] #[sqlx::test(fixtures("base", "lockfile_python"))] async fn test_extra_requirements_python2(db: Pool) { - let content = r#" -# py: ==3.11.11 + let content = r#"# py: ==3.11.11 # extra_requirements: # tiny==0.1.3 @@ -4048,8 +4045,7 @@ def main(): #[cfg(feature = "python")] #[sqlx::test(fixtures("base", "lockfile_python"))] async fn test_pins_python(db: Pool) { - let content = r#" -# py: ==3.11.11 + let content = r#"# py: ==3.11.11 # extra_requirements: # tiny==0.1.3 # bottle==0.13.2 diff --git a/backend/update_sqlx.sh b/backend/update_sqlx.sh index 11b5196ad8..09824d9e24 100755 --- a/backend/update_sqlx.sh +++ b/backend/update_sqlx.sh @@ -10,7 +10,6 @@ if [[ "$(uname)" == "Darwin" ]]; then fi cargo sqlx prepare --workspace -- --all-targets --all-features -./substitute_ee_code.sh -r --dir ../windmill-ee-private # Undo the samael changes on macOS if [[ "$(uname)" == "Darwin" ]]; then diff --git a/backend/windmill-api/Cargo.toml b/backend/windmill-api/Cargo.toml index 9f6de0c5ca..cfb175cb0f 100644 --- a/backend/windmill-api/Cargo.toml +++ b/backend/windmill-api/Cargo.toml @@ -10,6 +10,7 @@ path = "src/lib.rs" [features] default = [] +private = ["windmill-audit/private"] enterprise = ["windmill-queue/enterprise", "windmill-audit/enterprise", "windmill-git-sync/enterprise", "windmill-common/enterprise", "windmill-worker/enterprise"] stripe = [] agent_worker_server = [] diff --git a/backend/windmill-api/openapi-deref.json b/backend/windmill-api/openapi-deref.json index 91214e99b4..1186be6069 100644 --- a/backend/windmill-api/openapi-deref.json +++ b/backend/windmill-api/openapi-deref.json @@ -1,7 +1,7 @@ { "openapi": "3.0.3", "info": { - "version": "1.423.2", + "version": "1.492.1", "title": "Windmill API", "contact": { "name": "Windmill Team", @@ -197,6 +197,14 @@ }, { "$ref": "#/components/parameters/ActionKind" + }, + { + "name": "all_workspaces", + "in": "query", + "description": "get audit logs for all workspaces", + "schema": { + "type": "boolean" + } } ], "responses": { @@ -618,6 +626,9 @@ "is_super_admin": { "type": "boolean" }, + "is_devops": { + "type": "boolean" + }, "name": { "type": "string" } @@ -873,6 +884,27 @@ } } }, + "/github_app/connected_repositories": { + "get": { + "summary": "get connected repositories", + "operationId": "getGlobalConnectedRepositories", + "tags": [ + "git_sync" + ], + "responses": { + "200": { + "description": "connected repositories", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/GithubInstallations" + } + } + } + } + } + } + }, "/workspaces/list": { "get": { "summary": "list all workspaces visible to me", @@ -1329,9 +1361,24 @@ "content": { "application/json": { "schema": { - "type": "array", - "items": { - "$ref": "#/components/schemas/CriticalAlert" + "type": "object", + "properties": { + "alerts": { + "type": "array", + "items": { + "$ref": "#/components/schemas/CriticalAlert" + } + }, + "total_rows": { + "type": "integer", + "description": "Total number of rows matching the query.", + "example": 100 + }, + "total_pages": { + "type": "integer", + "description": "Total number of pages based on the page size.", + "example": 10 + } } } } @@ -1670,9 +1717,19 @@ "tags": [ "user" ], + "parameters": [ + { + "name": "if_expiring_in_less_than_s", + "in": "query", + "required": false, + "schema": { + "type": "integer" + } + } + ], "responses": { "200": { - "description": "free usage", + "description": "new token", "content": { "text/plain": { "schema": { @@ -1910,6 +1967,220 @@ } } }, + "/w/{workspace}/github_app/token": { + "post": { + "summary": "get github app token", + "operationId": "getGithubAppToken", + "tags": [ + "workspace" + ], + "parameters": [ + { + "$ref": "#/components/parameters/WorkspaceId" + } + ], + "requestBody": { + "description": "jwt job token", + "required": true, + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "job_token": { + "type": "string" + } + }, + "required": [ + "job_token" + ] + } + } + } + }, + "responses": { + "200": { + "description": "github app token", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "token": { + "type": "string" + } + }, + "required": [ + "token" + ] + } + } + } + } + } + } + }, + "/w/{workspace}/github_app/install_from_workspace": { + "post": { + "tags": [ + "Git Sync" + ], + "summary": "Install a GitHub installation from another workspace", + "operationId": "installFromWorkspace", + "parameters": [ + { + "$ref": "#/components/parameters/WorkspaceId" + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "source_workspace_id": { + "type": "string", + "description": "The ID of the workspace containing the installation to copy" + }, + "installation_id": { + "type": "number", + "description": "The ID of the GitHub installation to copy" + } + }, + "required": [ + "source_workspace_id", + "installation_id" + ] + } + } + } + }, + "responses": { + "200": { + "description": "Installation successfully copied" + } + } + } + }, + "/w/{workspace}/github_app/installation/{installation_id}": { + "delete": { + "summary": "Delete a GitHub installation from a workspace", + "operationId": "deleteFromWorkspace", + "description": "Removes a GitHub installation from the specified workspace. Requires admin privileges.", + "tags": [ + "Git Sync" + ], + "parameters": [ + { + "$ref": "#/components/parameters/WorkspaceId" + }, + { + "name": "installation_id", + "in": "path", + "required": true, + "schema": { + "type": "integer", + "format": "int64" + }, + "description": "The ID of the GitHub installation to delete" + } + ], + "responses": { + "200": { + "description": "Installation successfully deleted" + } + } + } + }, + "/w/{workspace}/github_app/export/{installationId}": { + "get": { + "summary": "Export GitHub installation JWT token", + "description": "Exports the JWT token for a specific GitHub installation in the workspace", + "operationId": "exportInstallation", + "tags": [ + "Git Sync" + ], + "parameters": [ + { + "name": "workspace", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "installationId", + "in": "path", + "required": true, + "schema": { + "type": "integer" + } + } + ], + "responses": { + "200": { + "description": "Successfully exported the JWT token", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "jwt_token": { + "type": "string" + } + } + } + } + } + } + } + } + }, + "/w/{workspace}/github_app/import": { + "post": { + "summary": "Import GitHub installation from JWT token", + "description": "Imports a GitHub installation from a JWT token exported from another instance", + "operationId": "importInstallation", + "tags": [ + "Git Sync" + ], + "parameters": [ + { + "name": "workspace", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "type": "object", + "required": [ + "jwt_token" + ], + "properties": { + "jwt_token": { + "type": "string" + } + } + } + } + } + }, + "responses": { + "200": { + "description": "Successfully imported the installation" + } + } + } + }, "/users/accept_invite": { "post": { "summary": "accept invite to workspace", @@ -2374,6 +2645,46 @@ } } }, + "/w/{workspace}/workspaces/change_workspace_color": { + "post": { + "summary": "change workspace id", + "operationId": "changeWorkspaceColor", + "tags": [ + "workspace" + ], + "parameters": [ + { + "$ref": "#/components/parameters/WorkspaceId" + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "color": { + "type": "string" + } + } + } + } + } + }, + "responses": { + "200": { + "description": "status", + "content": { + "text/plain": { + "schema": { + "type": "string" + } + } + } + } + } + } + }, "/w/{workspace}/users/whois/{username}": { "get": { "summary": "whois", @@ -2408,6 +2719,43 @@ } } }, + "/w/{workspace}/workspaces/operator_settings": { + "post": { + "operationId": "updateOperatorSettings", + "summary": "Update operator settings for a workspace", + "description": "Updates the operator settings for a specific workspace. Requires workspace admin privileges.", + "tags": [ + "workspace" + ], + "parameters": [ + { + "$ref": "#/components/parameters/WorkspaceId" + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/OperatorSettings" + } + } + } + }, + "responses": { + "200": { + "description": "Operator settings updated successfully", + "content": { + "text/plain": { + "schema": { + "type": "string" + } + } + } + } + } + } + }, "/users/exists/{email}": { "get": { "summary": "exists email", @@ -2540,6 +2888,15 @@ "slack_command_script": { "type": "string" }, + "teams_team_id": { + "type": "string" + }, + "teams_command_script": { + "type": "string" + }, + "teams_team_name": { + "type": "string" + }, "auto_invite_domain": { "type": "string" }, @@ -2552,9 +2909,6 @@ "plan": { "type": "string" }, - "automatic_billing": { - "type": "boolean" - }, "customer_id": { "type": "string" }, @@ -2564,11 +2918,8 @@ "deploy_to": { "type": "string" }, - "ai_resource": { - "$ref": "#/components/schemas/AiResource" - }, - "code_completion_enabled": { - "type": "boolean" + "ai_config": { + "$ref": "#/components/schemas/AIConfig" }, "error_handler": { "type": "string" @@ -2593,11 +2944,18 @@ }, "default_scripts": { "$ref": "#/components/schemas/WorkspaceDefaultScripts" + }, + "mute_critical_alerts": { + "type": "boolean" + }, + "color": { + "type": "string" + }, + "operator_settings": { + "$ref": "#/components/schemas/OperatorSettings" } }, "required": [ - "code_completion_enabled", - "automatic_billing", "error_handler_muted_on_cancel" ] } @@ -2690,16 +3048,16 @@ "usage": { "type": "number" }, - "seats": { - "type": "number" + "owner": { + "type": "string" }, - "automatic_billing": { - "type": "boolean" + "status": { + "type": "string" } }, "required": [ "premium", - "automatic_billing" + "owner" ] } } @@ -2708,10 +3066,43 @@ } } }, - "/w/{workspace}/workspaces/set_automatic_billing": { + "/w/{workspace}/workspaces/threshold_alert": { + "get": { + "summary": "get threshold alert info", + "operationId": "getThresholdAlert", + "tags": [ + "workspace" + ], + "parameters": [ + { + "$ref": "#/components/parameters/WorkspaceId" + } + ], + "responses": { + "200": { + "description": "status", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "threshold_alert_amount": { + "type": "number" + }, + "last_alert_sent": { + "type": "string", + "format": "date-time" + } + } + } + } + } + } + } + }, "post": { - "summary": "set automatic billing", - "operationId": "setAutomaticBilling", + "summary": "set threshold alert info", + "operationId": "setThresholdAlert", "tags": [ "workspace" ], @@ -2721,23 +3112,17 @@ } ], "requestBody": { - "description": "automatic billing", + "description": "threshold alert info", "required": true, "content": { "application/json": { "schema": { "type": "object", "properties": { - "automatic_billing": { - "type": "boolean" - }, - "seats": { + "threshold_alert_amount": { "type": "number" } - }, - "required": [ - "automatic_billing" - ] + } } } } @@ -2798,6 +3183,173 @@ } } }, + "/w/{workspace}/workspaces/edit_teams_command": { + "post": { + "summary": "edit teams command", + "operationId": "editTeamsCommand", + "tags": [ + "workspace" + ], + "parameters": [ + { + "$ref": "#/components/parameters/WorkspaceId" + } + ], + "requestBody": { + "description": "WorkspaceInvite", + "required": true, + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "slack_command_script": { + "type": "string" + } + } + } + } + } + }, + "responses": { + "200": { + "description": "status", + "content": { + "text/plain": { + "schema": { + "type": "string" + } + } + } + } + } + } + }, + "/w/{workspace}/workspaces/available_teams_ids": { + "get": { + "summary": "list available teams ids", + "operationId": "listAvailableTeamsIds", + "tags": [ + "workspace" + ], + "parameters": [ + { + "$ref": "#/components/parameters/WorkspaceId" + } + ], + "responses": { + "200": { + "description": "status", + "content": { + "application/json": { + "schema": { + "type": "array", + "items": { + "type": "object", + "properties": { + "team_name": { + "type": "string" + }, + "team_id": { + "type": "string" + } + } + } + } + } + } + } + } + } + }, + "/w/{workspace}/workspaces/available_teams_channels": { + "get": { + "summary": "list available teams channels", + "operationId": "listAvailableTeamsChannels", + "tags": [ + "workspace" + ], + "parameters": [ + { + "$ref": "#/components/parameters/WorkspaceId" + } + ], + "responses": { + "200": { + "description": "status", + "content": { + "application/json": { + "schema": { + "type": "array", + "items": { + "type": "object", + "properties": { + "channel_name": { + "type": "string" + }, + "channel_id": { + "type": "string" + }, + "service_url": { + "type": "string" + }, + "tenant_id": { + "type": "string" + } + } + } + } + } + } + } + } + } + }, + "/w/{workspace}/workspaces/connect_teams": { + "post": { + "summary": "connect teams", + "operationId": "connectTeams", + "tags": [ + "workspace" + ], + "parameters": [ + { + "$ref": "#/components/parameters/WorkspaceId" + } + ], + "requestBody": { + "description": "connect teams", + "required": true, + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "team_id": { + "type": "string" + }, + "team_name": { + "type": "string" + } + } + } + } + } + }, + "responses": { + "200": { + "description": "status", + "content": { + "text/plain": { + "schema": { + "type": "string" + } + } + } + } + } + } + }, "/w/{workspace}/workspaces/run_slack_message_test_job": { "post": { "summary": "run a job that sends a message to Slack", @@ -2851,6 +3403,59 @@ } } }, + "/w/{workspace}/workspaces/run_teams_message_test_job": { + "post": { + "summary": "run a job that sends a message to Teams", + "operationId": "runTeamsMessageTestJob", + "tags": [ + "workspace" + ], + "parameters": [ + { + "$ref": "#/components/parameters/WorkspaceId" + } + ], + "requestBody": { + "description": "path to hub script to run and its corresponding args", + "required": true, + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "hub_script_path": { + "type": "string" + }, + "channel": { + "type": "string" + }, + "test_msg": { + "type": "string" + } + } + } + } + } + }, + "responses": { + "200": { + "description": "status", + "content": { + "text/json": { + "schema": { + "type": "object", + "properties": { + "job_uuid": { + "type": "string" + } + } + } + } + } + } + } + } + }, "/w/{workspace}/workspaces/edit_deploy_to": { "post": { "summary": "edit deploy to", @@ -3000,18 +3605,7 @@ "content": { "application/json": { "schema": { - "type": "object", - "required": [ - "code_completion_enabled" - ], - "properties": { - "ai_resource": { - "$ref": "#/components/schemas/AiResource" - }, - "code_completion_enabled": { - "type": "boolean" - } - } + "$ref": "#/components/schemas/AIConfig" } } } @@ -3046,25 +3640,9 @@ "200": { "description": "status", "content": { - "text/plain": { + "application/json": { "schema": { - "type": "object", - "properties": { - "ai_provider": { - "type": "string" - }, - "exists_ai_resource": { - "type": "boolean" - }, - "code_completion_enabled": { - "type": "boolean" - } - }, - "required": [ - "ai_provider", - "exists_ai_resource", - "code_completion_enabled" - ] + "$ref": "#/components/schemas/AIConfig" } } } @@ -3578,11 +4156,35 @@ }, "websocket_used": { "type": "boolean" + }, + "kafka_used": { + "type": "boolean" + }, + "nats_used": { + "type": "boolean" + }, + "postgres_used": { + "type": "boolean" + }, + "mqtt_used": { + "type": "boolean" + }, + "gcp_used": { + "type": "boolean" + }, + "sqs_used": { + "type": "boolean" } }, "required": [ "http_routes_used", - "websocket_used" + "websocket_used", + "kafka_used", + "nats_used", + "postgres_used", + "mqtt_used", + "gcp_used", + "sqs_used" ] } } @@ -4211,6 +4813,186 @@ } } }, + "/w/{workspace}/workspaces/critical_alerts": { + "get": { + "summary": "Get all critical alerts for this workspace", + "operationId": "workspaceGetCriticalAlerts", + "tags": [ + "setting" + ], + "parameters": [ + { + "$ref": "#/components/parameters/WorkspaceId" + }, + { + "in": "query", + "name": "page", + "schema": { + "type": "integer", + "default": 1, + "description": "The page number to retrieve (minimum value is 1)" + } + }, + { + "in": "query", + "name": "page_size", + "schema": { + "type": "integer", + "default": 10, + "maximum": 100, + "description": "Number of alerts per page (maximum is 100)" + } + }, + { + "in": "query", + "name": "acknowledged", + "schema": { + "type": "boolean", + "nullable": true, + "description": "Filter by acknowledgment status; true for acknowledged, false for unacknowledged, and omit for all alerts" + } + } + ], + "responses": { + "200": { + "description": "Successfully retrieved all critical alerts", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "alerts": { + "type": "array", + "items": { + "$ref": "#/components/schemas/CriticalAlert" + } + }, + "total_rows": { + "type": "integer", + "description": "Total number of rows matching the query.", + "example": 100 + }, + "total_pages": { + "type": "integer", + "description": "Total number of pages based on the page size.", + "example": 10 + } + } + } + } + } + } + } + } + }, + "/w/{workspace}/workspaces/critical_alerts/{id}/acknowledge": { + "post": { + "summary": "Acknowledge a critical alert for this workspace", + "operationId": "workspaceAcknowledgeCriticalAlert", + "tags": [ + "setting" + ], + "parameters": [ + { + "$ref": "#/components/parameters/WorkspaceId" + }, + { + "in": "path", + "name": "id", + "required": true, + "schema": { + "type": "integer" + }, + "description": "The ID of the critical alert to acknowledge" + } + ], + "responses": { + "200": { + "description": "Successfully acknowledged the critical alert", + "content": { + "application/json": { + "schema": { + "type": "string", + "example": "Critical alert acknowledged" + } + } + } + } + } + } + }, + "/w/{workspace}/workspaces/critical_alerts/acknowledge_all": { + "post": { + "summary": "Acknowledge all unacknowledged critical alerts for this workspace", + "operationId": "workspaceAcknowledgeAllCriticalAlerts", + "tags": [ + "setting" + ], + "parameters": [ + { + "$ref": "#/components/parameters/WorkspaceId" + } + ], + "responses": { + "200": { + "description": "Successfully acknowledged all unacknowledged critical alerts.", + "content": { + "application/json": { + "schema": { + "type": "string", + "example": "All unacknowledged critical alerts acknowledged" + } + } + } + } + } + } + }, + "/w/{workspace}/workspaces/critical_alerts/mute": { + "post": { + "summary": "Mute critical alert UI for this workspace", + "operationId": "workspaceMuteCriticalAlertsUI", + "tags": [ + "setting" + ], + "parameters": [ + { + "$ref": "#/components/parameters/WorkspaceId" + } + ], + "requestBody": { + "description": "Boolean flag to mute critical alerts.", + "required": true, + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "mute_critical_alerts": { + "type": "boolean", + "description": "Whether critical alerts should be muted.", + "example": true + } + } + } + } + } + }, + "responses": { + "200": { + "description": "Successfully updated mute critical alert settings.", + "content": { + "application/json": { + "schema": { + "type": "string", + "example": "Updated mute critical alert UI settings for workspace: workspace_id" + } + } + } + } + } + } + }, "/oauth/login_callback/{client_name}": { "post": { "security": [], @@ -4562,6 +5344,32 @@ } } }, + "/w/{workspace}/oauth/disconnect_teams": { + "post": { + "summary": "disconnect teams", + "operationId": "disconnectTeams", + "tags": [ + "oauth" + ], + "parameters": [ + { + "$ref": "#/components/parameters/WorkspaceId" + } + ], + "responses": { + "200": { + "description": "disconnected teams", + "content": { + "text/plain": { + "schema": { + "type": "string" + } + } + } + } + } + } + }, "/oauth/list_logins": { "get": { "summary": "list oauth logins", @@ -4675,6 +5483,78 @@ } } }, + "/teams/sync": { + "post": { + "operationId": "syncTeams", + "summary": "synchronize Microsoft Teams information (teams/channels)", + "tags": [ + "teams" + ], + "responses": { + "200": { + "description": "Teams information successfully synchronized", + "content": { + "application/json": { + "schema": { + "type": "array", + "items": { + "$ref": "#/components/schemas/TeamInfo" + } + } + } + } + } + } + } + }, + "/teams/activities": { + "post": { + "summary": "send update to Microsoft Teams activity", + "description": "Respond to a Microsoft Teams activity after a workspace command is run", + "operationId": "sendMessageToConversation", + "tags": [ + "teams" + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "type": "object", + "required": [ + "conversation_id", + "text" + ], + "properties": { + "conversation_id": { + "type": "string", + "description": "The ID of the Teams conversation/activity" + }, + "success": { + "type": "boolean", + "description": "Used for styling the card conditionally", + "default": true + }, + "text": { + "type": "string", + "description": "The message text to be sent in the Teams card" + }, + "card_block": { + "type": "object", + "description": "The card block to be sent in the Teams card" + } + } + } + } + } + }, + "responses": { + "200": { + "description": "Activity processed successfully" + } + } + } + }, "/w/{workspace}/resources/create": { "post": { "summary": "create resource", @@ -5637,6 +6517,44 @@ } } }, + "/apps_u/public_app_by_custom_path/{custom_path}": { + "get": { + "summary": "get public app by custom path", + "operationId": "getPublicAppByCustomPath", + "tags": [ + "app" + ], + "parameters": [ + { + "$ref": "#/components/parameters/CustomPath" + } + ], + "responses": { + "200": { + "description": "app details", + "content": { + "application/json": { + "schema": { + "allOf": [ + { + "$ref": "#/components/schemas/AppWithLastVersion" + }, + { + "type": "object", + "properties": { + "workspace_id": { + "type": "string" + } + } + } + ] + } + } + } + } + } + } + }, "/scripts/hub/get/{path}": { "get": { "summary": "get hub script content by path", @@ -5986,7 +6904,7 @@ }, { "name": "last_parent_hash", - "description": "mask to filter scripts whom last parent in the chain has exact hash. \nBeware that each script stores only a limited number of parents. Hence\nthe last parent hash for a script is not necessarily its top-most parent.\nTo find the top-most parent you will have to jump from last to last hash\n until finding the parent\n", + "description": "mask to filter scripts whom last parent in the chain has exact hash.\nBeware that each script stores only a limited number of parents. Hence\nthe last parent hash for a script is not necessarily its top-most parent.\nTo find the top-most parent you will have to jump from last to last hash\n until finding the parent\n", "in": "query", "schema": { "type": "string" @@ -6002,7 +6920,7 @@ }, { "name": "show_archived", - "description": "(default false)\nshow only the archived files.\nwhen multiple archived hash share the same path, only the ones with the latest create_at\nare \ned.\n", + "description": "(default false)\nshow only the archived files.\nwhen multiple archived hash share the same path, only the ones with the latest create_at\nare\ned.\n", "in": "query", "schema": { "type": "boolean" @@ -6289,6 +7207,24 @@ "tags": [ "worker" ], + "parameters": [ + { + "name": "workspace", + "in": "query", + "schema": { + "type": "string" + }, + "required": false + }, + { + "name": "show_workspace_restriction", + "in": "query", + "schema": { + "type": "boolean" + }, + "required": false + } + ], "responses": { "200": { "description": "list of custom tags", @@ -6440,7 +7376,7 @@ }, "/w/{workspace}/scripts/delete/p/{path}": { "post": { - "summary": "delete all scripts at a given path (require admin)", + "summary": "delete script at a given path (require admin)", "operationId": "deleteScriptByPath", "tags": [ "script" @@ -6451,6 +7387,14 @@ }, { "$ref": "#/components/parameters/ScriptPath" + }, + { + "name": "keep_captures", + "description": "keep captures", + "in": "query", + "schema": { + "type": "boolean" + } } ], "responses": { @@ -6625,6 +7569,38 @@ } } }, + "/w/{workspace}/scripts/list_paths_from_workspace_runnable/{path}": { + "get": { + "summary": "list script paths using provided script as a relative import", + "operationId": "listScriptPathsFromWorkspaceRunnable", + "tags": [ + "script" + ], + "parameters": [ + { + "$ref": "#/components/parameters/WorkspaceId" + }, + { + "$ref": "#/components/parameters/ScriptPath" + } + ], + "responses": { + "200": { + "description": "list of script paths", + "content": { + "application/json": { + "schema": { + "type": "array", + "items": { + "type": "string" + } + } + } + } + } + } + } + }, "/w/{workspace}/scripts/get_latest_version/{path}": { "get": { "summary": "get scripts's latest version (hash)", @@ -6645,7 +7621,6 @@ "description": "Script version/hash", "content": { "application/json": { - "required": false, "schema": { "$ref": "#/components/schemas/ScriptHistory" } @@ -6895,6 +7870,96 @@ } } }, + "/w/{workspace}/jobs/list_selected_job_groups": { + "post": { + "summary": "list selected jobs script/flow schemas grouped by (kind, path)", + "operationId": "listSelectedJobGroups", + "tags": [ + "job" + ], + "parameters": [ + { + "$ref": "#/components/parameters/WorkspaceId" + } + ], + "requestBody": { + "description": "script args", + "required": true, + "content": { + "application/json": { + "schema": { + "type": "array", + "items": { + "type": "string", + "format": "uuid" + } + } + } + } + }, + "responses": { + "200": { + "description": "result", + "content": { + "text/plain": { + "schema": { + "type": "array", + "items": { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "script", + "flow" + ] + }, + "script_path": { + "type": "string" + }, + "latest_schema": { + "type": "object" + }, + "schemas": { + "type": "array", + "items": { + "type": "object", + "properties": { + "schema": { + "type": "object" + }, + "script_hash": { + "type": "string" + }, + "job_ids": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "schema", + "script_hash", + "job_ids" + ] + } + } + }, + "required": [ + "kind", + "script_path", + "latest_schema", + "schemas" + ] + } + } + } + } + } + } + } + }, "/w/{workspace}/jobs/run/p/{path}": { "post": { "summary": "run script by path", @@ -7491,7 +8556,6 @@ "description": "Flow version", "content": { "application/json": { - "required": false, "schema": { "$ref": "#/components/schemas/FlowVersion" } @@ -7501,6 +8565,41 @@ } } }, + "/w/{workspace}/flows/list_paths_from_workspace_runnable/{runnable_kind}/{path}": { + "get": { + "summary": "list flow paths from workspace runnable", + "operationId": "listFlowPathsFromWorkspaceRunnable", + "tags": [ + "flow" + ], + "parameters": [ + { + "$ref": "#/components/parameters/WorkspaceId" + }, + { + "$ref": "#/components/parameters/RunnableKind" + }, + { + "$ref": "#/components/parameters/ScriptPath" + } + ], + "responses": { + "200": { + "description": "list of flow paths", + "content": { + "application/json": { + "schema": { + "type": "array", + "items": { + "type": "string" + } + } + } + } + } + } + } + }, "/w/{workspace}/flows/get/v/{version}/p/{path}": { "get": { "summary": "get flow version", @@ -7510,7 +8609,6 @@ "$ref": "#/components/parameters/WorkspaceId" }, { - "type": "string", "name": "version", "in": "path", "required": true, @@ -7548,7 +8646,6 @@ "$ref": "#/components/parameters/WorkspaceId" }, { - "type": "string", "name": "version", "in": "path", "required": true, @@ -7632,6 +8729,40 @@ } } }, + "/w/{workspace}/flows/deployment_status/p/{path}": { + "get": { + "summary": "get flow deployment status", + "operationId": "getFlowDeploymentStatus", + "tags": [ + "flow" + ], + "parameters": [ + { + "$ref": "#/components/parameters/WorkspaceId" + }, + { + "$ref": "#/components/parameters/ScriptPath" + } + ], + "responses": { + "200": { + "description": "flow status", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "lock_error_logs": { + "type": "string" + } + } + } + } + } + } + } + } + }, "/w/{workspace}/flows/get_triggers_count/{path}": { "get": { "summary": "get triggers count of flow", @@ -7970,6 +9101,14 @@ }, { "$ref": "#/components/parameters/ScriptPath" + }, + { + "name": "keep_captures", + "description": "keep captures", + "in": "query", + "schema": { + "type": "boolean" + } } ], "responses": { @@ -8267,6 +9406,9 @@ }, "deployment_message": { "type": "string" + }, + "custom_path": { + "type": "string" } }, "required": [ @@ -8293,6 +9435,81 @@ } } }, + "/w/{workspace}/apps/create_raw": { + "post": { + "summary": "create app raw", + "operationId": "createAppRaw", + "tags": [ + "app" + ], + "parameters": [ + { + "$ref": "#/components/parameters/WorkspaceId" + } + ], + "requestBody": { + "description": "new app", + "required": true, + "content": { + "multipart/form-data": { + "schema": { + "type": "object", + "properties": { + "app": { + "type": "object", + "properties": { + "path": { + "type": "string" + }, + "value": {}, + "summary": { + "type": "string" + }, + "policy": { + "$ref": "#/components/schemas/Policy" + }, + "draft_only": { + "type": "boolean" + }, + "deployment_message": { + "type": "string" + }, + "custom_path": { + "type": "string" + } + }, + "required": [ + "path", + "value", + "summary", + "policy" + ] + }, + "js": { + "type": "string" + }, + "css": { + "type": "string" + } + } + } + } + } + }, + "responses": { + "201": { + "description": "app created", + "content": { + "text/plain": { + "schema": { + "type": "string" + } + } + } + } + } + } + }, "/w/{workspace}/apps/exists/{path}": { "get": { "summary": "does an app exisst at path", @@ -8358,6 +9575,35 @@ } } }, + "/w/{workspace}/apps/get/lite/{path}": { + "get": { + "summary": "get app lite by path", + "operationId": "getAppLiteByPath", + "tags": [ + "app" + ], + "parameters": [ + { + "$ref": "#/components/parameters/WorkspaceId" + }, + { + "$ref": "#/components/parameters/ScriptPath" + } + ], + "responses": { + "200": { + "description": "app lite details", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AppWithLastVersion" + } + } + } + } + } + } + }, "/w/{workspace}/apps/get/draft/{path}": { "get": { "summary": "get app by path with draft", @@ -8439,7 +9685,6 @@ "description": "App version", "content": { "application/json": { - "required": false, "schema": { "$ref": "#/components/schemas/AppHistory" } @@ -8449,6 +9694,41 @@ } } }, + "/w/{workspace}/apps/list_paths_from_workspace_runnable/{runnable_kind}/{path}": { + "get": { + "summary": "list app paths from workspace runnable", + "operationId": "listAppPathsFromWorkspaceRunnable", + "tags": [ + "app" + ], + "parameters": [ + { + "$ref": "#/components/parameters/WorkspaceId" + }, + { + "$ref": "#/components/parameters/RunnableKind" + }, + { + "$ref": "#/components/parameters/ScriptPath" + } + ], + "responses": { + "200": { + "description": "list of app paths", + "content": { + "application/json": { + "schema": { + "type": "array", + "items": { + "type": "string" + } + } + } + } + } + } + } + }, "/w/{workspace}/apps/history_update/a/{id}/v/{version}": { "post": { "summary": "update app history", @@ -8808,6 +10088,9 @@ }, "deployment_message": { "type": "string" + }, + "custom_path": { + "type": "string" } } } @@ -8828,6 +10111,155 @@ } } }, + "/w/{workspace}/apps/update_raw/{path}": { + "post": { + "summary": "update app", + "operationId": "updateAppRaw", + "tags": [ + "app" + ], + "parameters": [ + { + "$ref": "#/components/parameters/WorkspaceId" + }, + { + "$ref": "#/components/parameters/ScriptPath" + } + ], + "requestBody": { + "description": "update app", + "required": true, + "content": { + "multipart/form-data": { + "schema": { + "type": "object", + "properties": { + "app": { + "type": "object", + "properties": { + "path": { + "type": "string" + }, + "summary": { + "type": "string" + }, + "value": {}, + "policy": { + "$ref": "#/components/schemas/Policy" + }, + "deployment_message": { + "type": "string" + }, + "custom_path": { + "type": "string" + } + } + }, + "js": { + "type": "string" + }, + "css": { + "type": "string" + } + } + } + } + } + }, + "responses": { + "200": { + "description": "app updated", + "content": { + "text/plain": { + "schema": { + "type": "string" + } + } + } + } + } + } + }, + "/w/{workspace}/apps/custom_path_exists/{custom_path}": { + "get": { + "summary": "check if custom path exists", + "operationId": "customPathExists", + "tags": [ + "app" + ], + "parameters": [ + { + "$ref": "#/components/parameters/WorkspaceId" + }, + { + "$ref": "#/components/parameters/CustomPath" + } + ], + "responses": { + "200": { + "description": "custom path exists", + "content": { + "application/json": { + "schema": { + "type": "boolean" + } + } + } + } + } + } + }, + "/w/{workspace}/apps/sign_s3_objects": { + "post": { + "summary": "sign s3 objects, to be used by anonymous users in public apps", + "operationId": "signS3Objects", + "tags": [ + "app" + ], + "parameters": [ + { + "$ref": "#/components/parameters/WorkspaceId" + } + ], + "requestBody": { + "description": "s3 objects to sign", + "required": true, + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "s3_objects": { + "type": "array", + "items": { + "$ref": "#/components/schemas/S3Object" + } + } + }, + "required": [ + "s3_objects" + ] + } + } + } + }, + "responses": { + "200": { + "description": "signed s3 objects", + "content": { + "application/json": { + "schema": { + "type": "array", + "items": { + "$ref": "#/components/schemas/S3Object" + } + } + } + } + } + } + } + }, "/w/{workspace}/apps_u/execute_component/{path}": { "post": { "summary": "executeComponent", @@ -8857,6 +10289,9 @@ "path": { "type": "string" }, + "version": { + "type": "integer" + }, "args": {}, "raw_code": { "type": "object", @@ -8882,6 +10317,9 @@ "language" ] }, + "id": { + "type": "integer" + }, "force_viewer_static_fields": { "type": "object" }, @@ -8917,6 +10355,146 @@ } } }, + "/w/{workspace}/apps_u/upload_s3_file/{path}": { + "post": { + "summary": "upload s3 file from app", + "operationId": "uploadS3FileFromApp", + "tags": [ + "app" + ], + "parameters": [ + { + "$ref": "#/components/parameters/WorkspaceId" + }, + { + "$ref": "#/components/parameters/Path" + }, + { + "name": "file_key", + "in": "query", + "required": false, + "schema": { + "type": "string" + } + }, + { + "name": "file_extension", + "in": "query", + "required": false, + "schema": { + "type": "string" + } + }, + { + "name": "s3_resource_path", + "in": "query", + "required": false, + "schema": { + "type": "string" + } + }, + { + "name": "resource_type", + "in": "query", + "required": false, + "schema": { + "type": "string" + } + }, + { + "name": "storage", + "in": "query", + "schema": { + "type": "string" + } + }, + { + "name": "content_type", + "in": "query", + "schema": { + "type": "string" + } + }, + { + "name": "content_disposition", + "in": "query", + "schema": { + "type": "string" + } + } + ], + "requestBody": { + "description": "File content", + "required": true, + "content": { + "application/octet-stream": { + "schema": { + "type": "string", + "format": "binary" + } + } + } + }, + "responses": { + "200": { + "description": "file uploaded", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "file_key": { + "type": "string" + }, + "delete_token": { + "type": "string" + } + }, + "required": [ + "file_key", + "delete_token" + ] + } + } + } + } + } + } + }, + "/w/{workspace}/apps_u/delete_s3_file": { + "delete": { + "summary": "delete s3 file from app", + "operationId": "deleteS3FileFromApp", + "tags": [ + "app" + ], + "parameters": [ + { + "$ref": "#/components/parameters/WorkspaceId" + }, + { + "name": "delete_token", + "in": "query", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "file deleted", + "content": { + "text/plain": { + "schema": { + "type": "string" + } + } + } + } + } + } + }, "/w/{workspace}/jobs/run/f/{path}": { "post": { "summary": "run flow by path", @@ -9003,6 +10581,90 @@ } } }, + "/w/{workspace}/jobs/run/batch_rerun_jobs": { + "post": { + "summary": "re-run multiple jobs", + "operationId": "batchReRunJobs", + "tags": [ + "job" + ], + "parameters": [ + { + "$ref": "#/components/parameters/WorkspaceId" + } + ], + "requestBody": { + "description": "list of job ids to re run and arg tranforms", + "required": true, + "content": { + "application/json": { + "schema": { + "type": "object", + "required": [ + "job_ids", + "script_options_by_path", + "flow_options_by_path" + ], + "properties": { + "job_ids": { + "type": "array", + "items": { + "type": "string" + } + }, + "script_options_by_path": { + "type": "object", + "additionalProperties": { + "type": "object", + "properties": { + "input_transforms": { + "type": "object", + "additionalProperties": { + "$ref": "#/components/schemas/InputTransform" + } + }, + "use_latest_version": { + "type": "boolean" + } + } + } + }, + "flow_options_by_path": { + "type": "object", + "additionalProperties": { + "type": "object", + "properties": { + "input_transforms": { + "type": "object", + "additionalProperties": { + "$ref": "#/components/schemas/InputTransform" + } + }, + "use_latest_version": { + "type": "boolean" + } + } + } + } + } + } + } + } + }, + "responses": { + "201": { + "description": "stream of created job uuids separated by \\n. Lines may start with 'Error:'", + "content": { + "text/event-stream": { + "schema": { + "type": "string" + } + } + } + } + } + } + }, "/w/{workspace}/jobs/restart/f/{id}/from/{step_id}/{branch_or_iteration_n}": { "post": { "summary": "restart a completed flow at a given step", @@ -9426,6 +11088,9 @@ { "$ref": "#/components/parameters/ParentJob" }, + { + "$ref": "#/components/parameters/Worker" + }, { "$ref": "#/components/parameters/ScriptExactPath" }, @@ -9465,6 +11130,9 @@ { "$ref": "#/components/parameters/ResultFilter" }, + { + "$ref": "#/components/parameters/AllowWildcards" + }, { "$ref": "#/components/parameters/Tag" }, @@ -9587,10 +11255,213 @@ } } }, - "/w/{workspace}/jobs/queue/list_filtered_uuids": { + "/w/{workspace}/jobs/completed/count_jobs": { + "get": { + "summary": "count number of completed jobs with filter", + "operationId": "countCompletedJobs", + "tags": [ + "job" + ], + "parameters": [ + { + "$ref": "#/components/parameters/WorkspaceId" + }, + { + "name": "completed_after_s_ago", + "in": "query", + "schema": { + "type": "integer" + } + }, + { + "name": "success", + "in": "query", + "schema": { + "type": "boolean" + } + }, + { + "name": "tags", + "in": "query", + "schema": { + "type": "string" + } + }, + { + "name": "all_workspaces", + "in": "query", + "schema": { + "type": "boolean" + } + } + ], + "responses": { + "200": { + "description": "Count of completed jobs", + "content": { + "application/json": { + "schema": { + "type": "integer" + } + } + } + } + } + } + }, + "/w/{workspace}/jobs/list_filtered_uuids": { "get": { "summary": "get the ids of all jobs matching the given filters", - "operationId": "listFilteredUuids", + "operationId": "listFilteredJobsUuids", + "tags": [ + "job" + ], + "parameters": [ + { + "$ref": "#/components/parameters/WorkspaceId" + }, + { + "$ref": "#/components/parameters/CreatedBy" + }, + { + "$ref": "#/components/parameters/Label" + }, + { + "$ref": "#/components/parameters/Worker" + }, + { + "$ref": "#/components/parameters/ParentJob" + }, + { + "$ref": "#/components/parameters/ScriptExactPath" + }, + { + "$ref": "#/components/parameters/ScriptStartPath" + }, + { + "$ref": "#/components/parameters/SchedulePath" + }, + { + "$ref": "#/components/parameters/ScriptExactHash" + }, + { + "$ref": "#/components/parameters/StartedBefore" + }, + { + "$ref": "#/components/parameters/StartedAfter" + }, + { + "$ref": "#/components/parameters/CreatedBefore" + }, + { + "$ref": "#/components/parameters/CreatedAfter" + }, + { + "$ref": "#/components/parameters/CreatedOrStartedBefore" + }, + { + "$ref": "#/components/parameters/Running" + }, + { + "$ref": "#/components/parameters/ScheduledForBeforeNow" + }, + { + "$ref": "#/components/parameters/CreatedOrStartedAfter" + }, + { + "$ref": "#/components/parameters/CreatedOrStartedAfterCompletedJob" + }, + { + "$ref": "#/components/parameters/JobKinds" + }, + { + "$ref": "#/components/parameters/Suspended" + }, + { + "$ref": "#/components/parameters/ArgsFilter" + }, + { + "$ref": "#/components/parameters/Tag" + }, + { + "$ref": "#/components/parameters/ResultFilter" + }, + { + "$ref": "#/components/parameters/Page" + }, + { + "$ref": "#/components/parameters/PerPage" + }, + { + "name": "is_skipped", + "description": "is the job skipped", + "in": "query", + "schema": { + "type": "boolean" + } + }, + { + "name": "is_flow_step", + "description": "is the job a flow step", + "in": "query", + "schema": { + "type": "boolean" + } + }, + { + "name": "has_null_parent", + "description": "has null parent", + "in": "query", + "schema": { + "type": "boolean" + } + }, + { + "name": "success", + "description": "filter on successful jobs", + "in": "query", + "schema": { + "type": "boolean" + } + }, + { + "name": "all_workspaces", + "description": "get jobs from all workspaces (only valid if request come from the `admins` workspace)", + "in": "query", + "schema": { + "type": "boolean" + } + }, + { + "name": "is_not_schedule", + "description": "is not a scheduled job", + "in": "query", + "schema": { + "type": "boolean" + } + } + ], + "responses": { + "200": { + "description": "uuids of jobs", + "content": { + "application/json": { + "schema": { + "type": "array", + "items": { + "type": "string" + } + } + } + } + } + } + } + }, + "/w/{workspace}/jobs/queue/list_filtered_uuids": { + "get": { + "summary": "get the ids of all queued jobs matching the given filters", + "operationId": "listFilteredQueueUuids", "tags": [ "job" ], @@ -9646,6 +11517,9 @@ { "$ref": "#/components/parameters/ResultFilter" }, + { + "$ref": "#/components/parameters/AllowWildcards" + }, { "$ref": "#/components/parameters/Tag" }, @@ -9760,6 +11634,9 @@ { "$ref": "#/components/parameters/Label" }, + { + "$ref": "#/components/parameters/Worker" + }, { "$ref": "#/components/parameters/ParentJob" }, @@ -9793,6 +11670,9 @@ { "$ref": "#/components/parameters/ResultFilter" }, + { + "$ref": "#/components/parameters/AllowWildcards" + }, { "$ref": "#/components/parameters/Tag" }, @@ -9869,6 +11749,9 @@ { "$ref": "#/components/parameters/Label" }, + { + "$ref": "#/components/parameters/Worker" + }, { "$ref": "#/components/parameters/ParentJob" }, @@ -9926,6 +11809,9 @@ { "$ref": "#/components/parameters/ResultFilter" }, + { + "$ref": "#/components/parameters/AllowWildcards" + }, { "$ref": "#/components/parameters/Page" }, @@ -10294,7 +12180,9 @@ "description": "job log", "content": { "text/plain": { - "type": "string" + "schema": { + "type": "string" + } } } } @@ -10728,6 +12616,158 @@ } } }, + "/w/{workspace}/jobs/slack_approval/{id}": { + "get": { + "summary": "generate interactive slack approval for suspended job", + "operationId": "getSlackApprovalPayload", + "tags": [ + "job" + ], + "parameters": [ + { + "$ref": "#/components/parameters/WorkspaceId" + }, + { + "$ref": "#/components/parameters/JobId" + }, + { + "name": "approver", + "in": "query", + "schema": { + "type": "string" + } + }, + { + "name": "message", + "in": "query", + "schema": { + "type": "string" + } + }, + { + "name": "slack_resource_path", + "in": "query", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "channel_id", + "in": "query", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "flow_step_id", + "in": "query", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "default_args_json", + "in": "query", + "required": false, + "schema": { + "type": "string" + } + }, + { + "name": "dynamic_enums_json", + "in": "query", + "required": false, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "Interactive slack approval message sent successfully" + } + } + } + }, + "/w/{workspace}/jobs/teams_approval/{id}": { + "get": { + "summary": "generate interactive teams approval for suspended job", + "operationId": "getTeamsApprovalPayload", + "tags": [ + "job" + ], + "parameters": [ + { + "$ref": "#/components/parameters/WorkspaceId" + }, + { + "$ref": "#/components/parameters/JobId" + }, + { + "name": "approver", + "in": "query", + "schema": { + "type": "string" + } + }, + { + "name": "message", + "in": "query", + "schema": { + "type": "string" + } + }, + { + "name": "team_name", + "in": "query", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "channel_name", + "in": "query", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "flow_step_id", + "in": "query", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "default_args_json", + "in": "query", + "required": false, + "schema": { + "type": "string" + } + }, + { + "name": "dynamic_enums_json", + "in": "query", + "required": false, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "Interactive slack approval message sent successfully" + } + } + } + }, "/w/{workspace}/jobs_u/resume/{id}/{resume_id}/{signature}": { "get": { "summary": "resume a job for a suspended flow", @@ -11172,6 +13212,9 @@ }, "timezone": { "type": "string" + }, + "cron_version": { + "type": "string" } }, "required": [ @@ -11823,10 +13866,15 @@ "delete", "patch" ] + }, + "trigger_path": { + "type": "string" + }, + "workspaced_route": { + "type": "boolean" } }, "required": [ - "kind", "route_path", "http_method" ] @@ -12118,6 +14166,2452 @@ } } }, + "/w/{workspace}/websocket_triggers/test": { + "post": { + "summary": "test websocket connection", + "operationId": "testWebsocketConnection", + "tags": [ + "websocket_trigger" + ], + "parameters": [ + { + "$ref": "#/components/parameters/WorkspaceId" + } + ], + "requestBody": { + "description": "test websocket connection", + "required": true, + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "url": { + "type": "string" + }, + "url_runnable_args": { + "$ref": "#/components/schemas/ScriptArgs" + }, + "can_return_message": { + "type": "boolean" + } + }, + "required": [ + "url", + "can_return_message" + ] + } + } + } + }, + "responses": { + "200": { + "description": "successfuly connected to websocket", + "content": { + "text/plain": { + "schema": { + "type": "string" + } + } + } + } + } + } + }, + "/w/{workspace}/kafka_triggers/create": { + "post": { + "summary": "create kafka trigger", + "operationId": "createKafkaTrigger", + "tags": [ + "kafka_trigger" + ], + "parameters": [ + { + "$ref": "#/components/parameters/WorkspaceId" + } + ], + "requestBody": { + "description": "new kafka trigger", + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/NewKafkaTrigger" + } + } + } + }, + "responses": { + "201": { + "description": "kafka trigger created", + "content": { + "text/plain": { + "schema": { + "type": "string" + } + } + } + } + } + } + }, + "/w/{workspace}/kafka_triggers/update/{path}": { + "post": { + "summary": "update kafka trigger", + "operationId": "updateKafkaTrigger", + "tags": [ + "kafka_trigger" + ], + "parameters": [ + { + "$ref": "#/components/parameters/WorkspaceId" + }, + { + "$ref": "#/components/parameters/Path" + } + ], + "requestBody": { + "description": "updated trigger", + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/EditKafkaTrigger" + } + } + } + }, + "responses": { + "200": { + "description": "kafka trigger updated", + "content": { + "text/plain": { + "schema": { + "type": "string" + } + } + } + } + } + } + }, + "/w/{workspace}/kafka_triggers/delete/{path}": { + "delete": { + "summary": "delete kafka trigger", + "operationId": "deleteKafkaTrigger", + "tags": [ + "kafka_trigger" + ], + "parameters": [ + { + "$ref": "#/components/parameters/WorkspaceId" + }, + { + "$ref": "#/components/parameters/Path" + } + ], + "responses": { + "200": { + "description": "kafka trigger deleted", + "content": { + "text/plain": { + "schema": { + "type": "string" + } + } + } + } + } + } + }, + "/w/{workspace}/kafka_triggers/get/{path}": { + "get": { + "summary": "get kafka trigger", + "operationId": "getKafkaTrigger", + "tags": [ + "kafka_trigger" + ], + "parameters": [ + { + "$ref": "#/components/parameters/WorkspaceId" + }, + { + "$ref": "#/components/parameters/Path" + } + ], + "responses": { + "200": { + "description": "kafka trigger deleted", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/KafkaTrigger" + } + } + } + } + } + } + }, + "/w/{workspace}/kafka_triggers/list": { + "get": { + "summary": "list kafka triggers", + "operationId": "listKafkaTriggers", + "tags": [ + "kafka_trigger" + ], + "parameters": [ + { + "$ref": "#/components/parameters/WorkspaceId", + "required": true + }, + { + "$ref": "#/components/parameters/Page" + }, + { + "$ref": "#/components/parameters/PerPage" + }, + { + "name": "path", + "description": "filter by path", + "in": "query", + "schema": { + "type": "string" + } + }, + { + "name": "is_flow", + "in": "query", + "schema": { + "type": "boolean" + } + }, + { + "name": "path_start", + "in": "query", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "kafka trigger list", + "content": { + "application/json": { + "schema": { + "type": "array", + "items": { + "$ref": "#/components/schemas/KafkaTrigger" + } + } + } + } + } + } + } + }, + "/w/{workspace}/kafka_triggers/exists/{path}": { + "get": { + "summary": "does kafka trigger exists", + "operationId": "existsKafkaTrigger", + "tags": [ + "kafka_trigger" + ], + "parameters": [ + { + "$ref": "#/components/parameters/WorkspaceId" + }, + { + "$ref": "#/components/parameters/Path" + } + ], + "responses": { + "200": { + "description": "kafka trigger exists", + "content": { + "application/json": { + "schema": { + "type": "boolean" + } + } + } + } + } + } + }, + "/w/{workspace}/kafka_triggers/setenabled/{path}": { + "post": { + "summary": "set enabled kafka trigger", + "operationId": "setKafkaTriggerEnabled", + "tags": [ + "kafka_trigger" + ], + "parameters": [ + { + "$ref": "#/components/parameters/WorkspaceId" + }, + { + "$ref": "#/components/parameters/Path" + } + ], + "requestBody": { + "description": "updated kafka trigger enable", + "required": true, + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "enabled": { + "type": "boolean" + } + }, + "required": [ + "enabled" + ] + } + } + } + }, + "responses": { + "200": { + "description": "kafka trigger enabled set", + "content": { + "text/plain": { + "schema": { + "type": "string" + } + } + } + } + } + } + }, + "/w/{workspace}/kafka_triggers/test": { + "post": { + "summary": "test kafka connection", + "operationId": "testKafkaConnection", + "tags": [ + "kafka_trigger" + ], + "parameters": [ + { + "$ref": "#/components/parameters/WorkspaceId" + } + ], + "requestBody": { + "description": "test kafka connection", + "required": true, + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "connection": { + "type": "object" + } + }, + "required": [ + "connection" + ] + } + } + } + }, + "responses": { + "200": { + "description": "successfuly connected to kafka brokers", + "content": { + "text/plain": { + "schema": { + "type": "string" + } + } + } + } + } + } + }, + "/w/{workspace}/nats_triggers/create": { + "post": { + "summary": "create nats trigger", + "operationId": "createNatsTrigger", + "tags": [ + "nats_trigger" + ], + "parameters": [ + { + "$ref": "#/components/parameters/WorkspaceId" + } + ], + "requestBody": { + "description": "new nats trigger", + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/NewNatsTrigger" + } + } + } + }, + "responses": { + "201": { + "description": "nats trigger created", + "content": { + "text/plain": { + "schema": { + "type": "string" + } + } + } + } + } + } + }, + "/w/{workspace}/nats_triggers/update/{path}": { + "post": { + "summary": "update nats trigger", + "operationId": "updateNatsTrigger", + "tags": [ + "nats_trigger" + ], + "parameters": [ + { + "$ref": "#/components/parameters/WorkspaceId" + }, + { + "$ref": "#/components/parameters/Path" + } + ], + "requestBody": { + "description": "updated trigger", + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/EditNatsTrigger" + } + } + } + }, + "responses": { + "200": { + "description": "nats trigger updated", + "content": { + "text/plain": { + "schema": { + "type": "string" + } + } + } + } + } + } + }, + "/w/{workspace}/nats_triggers/delete/{path}": { + "delete": { + "summary": "delete nats trigger", + "operationId": "deleteNatsTrigger", + "tags": [ + "nats_trigger" + ], + "parameters": [ + { + "$ref": "#/components/parameters/WorkspaceId" + }, + { + "$ref": "#/components/parameters/Path" + } + ], + "responses": { + "200": { + "description": "nats trigger deleted", + "content": { + "text/plain": { + "schema": { + "type": "string" + } + } + } + } + } + } + }, + "/w/{workspace}/nats_triggers/get/{path}": { + "get": { + "summary": "get nats trigger", + "operationId": "getNatsTrigger", + "tags": [ + "nats_trigger" + ], + "parameters": [ + { + "$ref": "#/components/parameters/WorkspaceId" + }, + { + "$ref": "#/components/parameters/Path" + } + ], + "responses": { + "200": { + "description": "nats trigger deleted", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/NatsTrigger" + } + } + } + } + } + } + }, + "/w/{workspace}/nats_triggers/list": { + "get": { + "summary": "list nats triggers", + "operationId": "listNatsTriggers", + "tags": [ + "nats_trigger" + ], + "parameters": [ + { + "$ref": "#/components/parameters/WorkspaceId", + "required": true + }, + { + "$ref": "#/components/parameters/Page" + }, + { + "$ref": "#/components/parameters/PerPage" + }, + { + "name": "path", + "description": "filter by path", + "in": "query", + "schema": { + "type": "string" + } + }, + { + "name": "is_flow", + "in": "query", + "schema": { + "type": "boolean" + } + }, + { + "name": "path_start", + "in": "query", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "nats trigger list", + "content": { + "application/json": { + "schema": { + "type": "array", + "items": { + "$ref": "#/components/schemas/NatsTrigger" + } + } + } + } + } + } + } + }, + "/w/{workspace}/nats_triggers/exists/{path}": { + "get": { + "summary": "does nats trigger exists", + "operationId": "existsNatsTrigger", + "tags": [ + "nats_trigger" + ], + "parameters": [ + { + "$ref": "#/components/parameters/WorkspaceId" + }, + { + "$ref": "#/components/parameters/Path" + } + ], + "responses": { + "200": { + "description": "nats trigger exists", + "content": { + "application/json": { + "schema": { + "type": "boolean" + } + } + } + } + } + } + }, + "/w/{workspace}/nats_triggers/setenabled/{path}": { + "post": { + "summary": "set enabled nats trigger", + "operationId": "setNatsTriggerEnabled", + "tags": [ + "nats_trigger" + ], + "parameters": [ + { + "$ref": "#/components/parameters/WorkspaceId" + }, + { + "$ref": "#/components/parameters/Path" + } + ], + "requestBody": { + "description": "updated nats trigger enable", + "required": true, + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "enabled": { + "type": "boolean" + } + }, + "required": [ + "enabled" + ] + } + } + } + }, + "responses": { + "200": { + "description": "nats trigger enabled set", + "content": { + "text/plain": { + "schema": { + "type": "string" + } + } + } + } + } + } + }, + "/w/{workspace}/nats_triggers/test": { + "post": { + "summary": "test NATS connection", + "operationId": "testNatsConnection", + "tags": [ + "nats_trigger" + ], + "parameters": [ + { + "$ref": "#/components/parameters/WorkspaceId" + } + ], + "requestBody": { + "description": "test nats connection", + "required": true, + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "connection": { + "type": "object" + } + }, + "required": [ + "connection" + ] + } + } + } + }, + "responses": { + "200": { + "description": "successfuly connected to NATS servers", + "content": { + "text/plain": { + "schema": { + "type": "string" + } + } + } + } + } + } + }, + "/w/{workspace}/sqs_triggers/create": { + "post": { + "summary": "create sqs trigger", + "operationId": "createSqsTrigger", + "tags": [ + "sqs_trigger" + ], + "parameters": [ + { + "$ref": "#/components/parameters/WorkspaceId" + } + ], + "requestBody": { + "description": "new sqs trigger", + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/NewSqsTrigger" + } + } + } + }, + "responses": { + "201": { + "description": "sqs trigger created", + "content": { + "text/plain": { + "schema": { + "type": "string" + } + } + } + } + } + } + }, + "/w/{workspace}/sqs_triggers/update/{path}": { + "post": { + "summary": "update sqs trigger", + "operationId": "updateSqsTrigger", + "tags": [ + "sqs_trigger" + ], + "parameters": [ + { + "$ref": "#/components/parameters/WorkspaceId" + }, + { + "$ref": "#/components/parameters/Path" + } + ], + "requestBody": { + "description": "updated trigger", + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/EditSqsTrigger" + } + } + } + }, + "responses": { + "200": { + "description": "sqs trigger updated", + "content": { + "text/plain": { + "schema": { + "type": "string" + } + } + } + } + } + } + }, + "/w/{workspace}/sqs_triggers/delete/{path}": { + "delete": { + "summary": "delete sqs trigger", + "operationId": "deleteSqsTrigger", + "tags": [ + "sqs_trigger" + ], + "parameters": [ + { + "$ref": "#/components/parameters/WorkspaceId" + }, + { + "$ref": "#/components/parameters/Path" + } + ], + "responses": { + "200": { + "description": "sqs trigger deleted", + "content": { + "text/plain": { + "schema": { + "type": "string" + } + } + } + } + } + } + }, + "/w/{workspace}/sqs_triggers/get/{path}": { + "get": { + "summary": "get sqs trigger", + "operationId": "getSqsTrigger", + "tags": [ + "sqs_trigger" + ], + "parameters": [ + { + "$ref": "#/components/parameters/WorkspaceId" + }, + { + "$ref": "#/components/parameters/Path" + } + ], + "responses": { + "200": { + "description": "sqs trigger deleted", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SqsTrigger" + } + } + } + } + } + } + }, + "/w/{workspace}/sqs_triggers/list": { + "get": { + "summary": "list sqs triggers", + "operationId": "listSqsTriggers", + "tags": [ + "sqs_trigger" + ], + "parameters": [ + { + "$ref": "#/components/parameters/WorkspaceId", + "required": true + }, + { + "$ref": "#/components/parameters/Page" + }, + { + "$ref": "#/components/parameters/PerPage" + }, + { + "name": "path", + "description": "filter by path", + "in": "query", + "schema": { + "type": "string" + } + }, + { + "name": "is_flow", + "in": "query", + "schema": { + "type": "boolean" + } + }, + { + "name": "path_start", + "in": "query", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "sqs trigger list", + "content": { + "application/json": { + "schema": { + "type": "array", + "items": { + "$ref": "#/components/schemas/SqsTrigger" + } + } + } + } + } + } + } + }, + "/w/{workspace}/sqs_triggers/exists/{path}": { + "get": { + "summary": "does sqs trigger exists", + "operationId": "existsSqsTrigger", + "tags": [ + "sqs_trigger" + ], + "parameters": [ + { + "$ref": "#/components/parameters/WorkspaceId" + }, + { + "$ref": "#/components/parameters/Path" + } + ], + "responses": { + "200": { + "description": "sqs trigger exists", + "content": { + "application/json": { + "schema": { + "type": "boolean" + } + } + } + } + } + } + }, + "/w/{workspace}/sqs_triggers/setenabled/{path}": { + "post": { + "summary": "set enabled sqs trigger", + "operationId": "setSqsTriggerEnabled", + "tags": [ + "sqs_trigger" + ], + "parameters": [ + { + "$ref": "#/components/parameters/WorkspaceId" + }, + { + "$ref": "#/components/parameters/Path" + } + ], + "requestBody": { + "description": "updated sqs trigger enable", + "required": true, + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "enabled": { + "type": "boolean" + } + }, + "required": [ + "enabled" + ] + } + } + } + }, + "responses": { + "200": { + "description": "sqs trigger enabled set", + "content": { + "text/plain": { + "schema": { + "type": "string" + } + } + } + } + } + } + }, + "/w/{workspace}/sqs_triggers/test": { + "post": { + "summary": "test sqs connection", + "operationId": "testSqsConnection", + "tags": [ + "sqs_trigger" + ], + "parameters": [ + { + "$ref": "#/components/parameters/WorkspaceId" + } + ], + "requestBody": { + "description": "test sqs connection", + "required": true, + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "connection": { + "type": "object" + } + }, + "required": [ + "connection" + ] + } + } + } + }, + "responses": { + "200": { + "description": "successfuly connected to sqs", + "content": { + "text/plain": { + "schema": { + "type": "string" + } + } + } + } + } + } + }, + "/w/{workspace}/mqtt_triggers/create": { + "post": { + "summary": "create mqtt trigger", + "operationId": "createMqttTrigger", + "tags": [ + "mqtt_trigger" + ], + "parameters": [ + { + "$ref": "#/components/parameters/WorkspaceId" + } + ], + "requestBody": { + "description": "new mqtt trigger", + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/NewMqttTrigger" + } + } + } + }, + "responses": { + "201": { + "description": "mqtt trigger created", + "content": { + "text/plain": { + "schema": { + "type": "string" + } + } + } + } + } + } + }, + "/w/{workspace}/mqtt_triggers/update/{path}": { + "post": { + "summary": "update mqtt trigger", + "operationId": "updateMqttTrigger", + "tags": [ + "mqtt_trigger" + ], + "parameters": [ + { + "$ref": "#/components/parameters/WorkspaceId" + }, + { + "$ref": "#/components/parameters/Path" + } + ], + "requestBody": { + "description": "updated trigger", + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/EditMqttTrigger" + } + } + } + }, + "responses": { + "200": { + "description": "mqtt trigger updated", + "content": { + "text/plain": { + "schema": { + "type": "string" + } + } + } + } + } + } + }, + "/w/{workspace}/mqtt_triggers/delete/{path}": { + "delete": { + "summary": "delete mqtt trigger", + "operationId": "deleteMqttTrigger", + "tags": [ + "mqtt_trigger" + ], + "parameters": [ + { + "$ref": "#/components/parameters/WorkspaceId" + }, + { + "$ref": "#/components/parameters/Path" + } + ], + "responses": { + "200": { + "description": "mqtt trigger deleted", + "content": { + "text/plain": { + "schema": { + "type": "string" + } + } + } + } + } + } + }, + "/w/{workspace}/mqtt_triggers/get/{path}": { + "get": { + "summary": "get mqtt trigger", + "operationId": "getMqttTrigger", + "tags": [ + "mqtt_trigger" + ], + "parameters": [ + { + "$ref": "#/components/parameters/WorkspaceId" + }, + { + "$ref": "#/components/parameters/Path" + } + ], + "responses": { + "200": { + "description": "mqtt trigger deleted", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/MqttTrigger" + } + } + } + } + } + } + }, + "/w/{workspace}/mqtt_triggers/list": { + "get": { + "summary": "list mqtt triggers", + "operationId": "listMqttTriggers", + "tags": [ + "mqtt_trigger" + ], + "parameters": [ + { + "$ref": "#/components/parameters/WorkspaceId", + "required": true + }, + { + "$ref": "#/components/parameters/Page" + }, + { + "$ref": "#/components/parameters/PerPage" + }, + { + "name": "path", + "description": "filter by path", + "in": "query", + "schema": { + "type": "string" + } + }, + { + "name": "is_flow", + "in": "query", + "schema": { + "type": "boolean" + } + }, + { + "name": "path_start", + "in": "query", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "mqtt trigger list", + "content": { + "application/json": { + "schema": { + "type": "array", + "items": { + "$ref": "#/components/schemas/MqttTrigger" + } + } + } + } + } + } + } + }, + "/w/{workspace}/mqtt_triggers/exists/{path}": { + "get": { + "summary": "does mqtt trigger exists", + "operationId": "existsMqttTrigger", + "tags": [ + "mqtt_trigger" + ], + "parameters": [ + { + "$ref": "#/components/parameters/WorkspaceId" + }, + { + "$ref": "#/components/parameters/Path" + } + ], + "responses": { + "200": { + "description": "mqtt trigger exists", + "content": { + "application/json": { + "schema": { + "type": "boolean" + } + } + } + } + } + } + }, + "/w/{workspace}/mqtt_triggers/setenabled/{path}": { + "post": { + "summary": "set enabled mqtt trigger", + "operationId": "setMqttTriggerEnabled", + "tags": [ + "mqtt_trigger" + ], + "parameters": [ + { + "$ref": "#/components/parameters/WorkspaceId" + }, + { + "$ref": "#/components/parameters/Path" + } + ], + "requestBody": { + "description": "updated mqtt trigger enable", + "required": true, + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "enabled": { + "type": "boolean" + } + }, + "required": [ + "enabled" + ] + } + } + } + }, + "responses": { + "200": { + "description": "mqtt trigger enabled set", + "content": { + "text/plain": { + "schema": { + "type": "string" + } + } + } + } + } + } + }, + "/w/{workspace}/mqtt_triggers/test": { + "post": { + "summary": "test mqtt connection", + "operationId": "testMqttConnection", + "tags": [ + "mqtt_trigger" + ], + "parameters": [ + { + "$ref": "#/components/parameters/WorkspaceId" + } + ], + "requestBody": { + "description": "test mqtt connection", + "required": true, + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "connection": { + "type": "object" + } + }, + "required": [ + "connection" + ] + } + } + } + }, + "responses": { + "200": { + "description": "successfully connected to mqtt", + "content": { + "text/plain": { + "schema": { + "type": "string" + } + } + } + } + } + } + }, + "/w/{workspace}/gcp_triggers/create": { + "post": { + "summary": "create gcp trigger", + "operationId": "createGcpTrigger", + "tags": [ + "gcp_trigger" + ], + "parameters": [ + { + "$ref": "#/components/parameters/WorkspaceId" + } + ], + "requestBody": { + "description": "new gcp trigger", + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/GcpTriggerData" + } + } + } + }, + "responses": { + "201": { + "description": "gcp trigger created", + "content": { + "text/plain": { + "schema": { + "type": "string" + } + } + } + } + } + } + }, + "/w/{workspace}/gcp_triggers/update/{path}": { + "post": { + "summary": "update gcp trigger", + "operationId": "updateGcpTrigger", + "tags": [ + "gcp_trigger" + ], + "parameters": [ + { + "$ref": "#/components/parameters/WorkspaceId" + }, + { + "$ref": "#/components/parameters/Path" + } + ], + "requestBody": { + "description": "updated trigger", + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/GcpTriggerData" + } + } + } + }, + "responses": { + "200": { + "description": "gcp trigger updated", + "content": { + "text/plain": { + "schema": { + "type": "string" + } + } + } + } + } + } + }, + "/w/{workspace}/gcp_triggers/delete/{path}": { + "delete": { + "summary": "delete gcp trigger", + "operationId": "deleteGcpTrigger", + "tags": [ + "gcp_trigger" + ], + "parameters": [ + { + "$ref": "#/components/parameters/WorkspaceId" + }, + { + "$ref": "#/components/parameters/Path" + } + ], + "responses": { + "200": { + "description": "gcp trigger deleted", + "content": { + "text/plain": { + "schema": { + "type": "string" + } + } + } + } + } + } + }, + "/w/{workspace}/gcp_triggers/get/{path}": { + "get": { + "summary": "get gcp trigger", + "operationId": "getGcpTrigger", + "tags": [ + "gcp_trigger" + ], + "parameters": [ + { + "$ref": "#/components/parameters/WorkspaceId" + }, + { + "$ref": "#/components/parameters/Path" + } + ], + "responses": { + "200": { + "description": "gcp trigger deleted", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/GcpTrigger" + } + } + } + } + } + } + }, + "/w/{workspace}/gcp_triggers/list": { + "get": { + "summary": "list gcp triggers", + "operationId": "listGcpTriggers", + "tags": [ + "gcp_trigger" + ], + "parameters": [ + { + "$ref": "#/components/parameters/WorkspaceId", + "required": true + }, + { + "$ref": "#/components/parameters/Page" + }, + { + "$ref": "#/components/parameters/PerPage" + }, + { + "name": "path", + "description": "filter by path", + "in": "query", + "schema": { + "type": "string" + } + }, + { + "name": "is_flow", + "in": "query", + "schema": { + "type": "boolean" + } + }, + { + "name": "path_start", + "in": "query", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "gcp trigger list", + "content": { + "application/json": { + "schema": { + "type": "array", + "items": { + "$ref": "#/components/schemas/GcpTrigger" + } + } + } + } + } + } + } + }, + "/w/{workspace}/gcp_triggers/exists/{path}": { + "get": { + "summary": "does gcp trigger exists", + "operationId": "existsGcpTrigger", + "tags": [ + "gcp_trigger" + ], + "parameters": [ + { + "$ref": "#/components/parameters/WorkspaceId" + }, + { + "$ref": "#/components/parameters/Path" + } + ], + "responses": { + "200": { + "description": "gcp trigger exists", + "content": { + "application/json": { + "schema": { + "type": "boolean" + } + } + } + } + } + } + }, + "/w/{workspace}/gcp_triggers/setenabled/{path}": { + "post": { + "summary": "set enabled gcp trigger", + "operationId": "setGcpTriggerEnabled", + "tags": [ + "gcp_trigger" + ], + "parameters": [ + { + "$ref": "#/components/parameters/WorkspaceId" + }, + { + "$ref": "#/components/parameters/Path" + } + ], + "requestBody": { + "description": "updated gcp trigger enable", + "required": true, + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "enabled": { + "type": "boolean" + } + }, + "required": [ + "enabled" + ] + } + } + } + }, + "responses": { + "200": { + "description": "gcp trigger enabled set", + "content": { + "text/plain": { + "schema": { + "type": "string" + } + } + } + } + } + } + }, + "/w/{workspace}/gcp_triggers/test": { + "post": { + "summary": "test gcp connection", + "operationId": "testGcpConnection", + "tags": [ + "gcp_trigger" + ], + "parameters": [ + { + "$ref": "#/components/parameters/WorkspaceId" + } + ], + "requestBody": { + "description": "test gcp connection", + "required": true, + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "connection": { + "type": "object" + } + }, + "required": [ + "connection" + ] + } + } + } + }, + "responses": { + "200": { + "description": "try to connect to a gcp broker", + "content": { + "text/plain": { + "schema": { + "type": "string" + } + } + } + } + } + } + }, + "/w/{workspace}/gcp_triggers/subscriptions/delete/{path}": { + "delete": { + "summary": "delete gcp trigger", + "operationId": "deleteGcpSubscription", + "tags": [ + "gcp_trigger" + ], + "parameters": [ + { + "$ref": "#/components/parameters/WorkspaceId" + }, + { + "$ref": "#/components/parameters/Path" + } + ], + "requestBody": { + "description": "args to delete subscription from google cloud", + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/DeleteGcpSubscription" + } + } + } + }, + "responses": { + "200": { + "description": "gcp trigger deleted", + "content": { + "text/plain": { + "schema": { + "type": "string" + } + } + } + } + } + } + }, + "/w/{workspace}/gcp_triggers/topics/list/{path}": { + "get": { + "summary": "list all topics of google cloud service", + "operationId": "listGoogleTopics", + "tags": [ + "gcp_trigger" + ], + "parameters": [ + { + "$ref": "#/components/parameters/WorkspaceId" + }, + { + "$ref": "#/components/parameters/Path" + } + ], + "responses": { + "200": { + "description": "get all google topics", + "content": { + "application/json": { + "schema": { + "type": "array", + "items": { + "type": "string" + } + } + } + } + } + } + } + }, + "/w/{workspace}/gcp_triggers/subscriptions/list/{path}": { + "post": { + "summary": "list all subscription of a give topic from google cloud service", + "operationId": "listAllTGoogleTopicSubscriptions", + "tags": [ + "gcp_trigger" + ], + "parameters": [ + { + "$ref": "#/components/parameters/WorkspaceId" + }, + { + "$ref": "#/components/parameters/Path" + } + ], + "requestBody": { + "description": "args to get subscription's topic from google cloud", + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/GetAllTopicSubscription" + } + } + } + }, + "responses": { + "200": { + "description": "get all google topic subscriptions name", + "content": { + "application/json": { + "schema": { + "type": "array", + "items": { + "type": "string" + } + } + } + } + } + } + } + }, + "/w/{workspace}/postgres_triggers/is_valid_postgres_configuration/{path}": { + "get": { + "summary": "check if postgres configuration is set to logical", + "operationId": "isValidPostgresConfiguration", + "tags": [ + "postgres_trigger" + ], + "parameters": [ + { + "$ref": "#/components/parameters/WorkspaceId" + }, + { + "$ref": "#/components/parameters/Path" + } + ], + "responses": { + "200": { + "description": "boolean that indicates if postgres is set to logical level or not", + "content": { + "application/json": { + "schema": { + "type": "boolean" + } + } + } + } + } + } + }, + "/w/{workspace}/postgres_triggers/create_template_script": { + "post": { + "summary": "create template script", + "operationId": "createTemplateScript", + "tags": [ + "postgres_trigger" + ], + "parameters": [ + { + "$ref": "#/components/parameters/WorkspaceId" + } + ], + "requestBody": { + "description": "template script", + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/TemplateScript" + } + } + } + }, + "responses": { + "200": { + "description": "custom id to retrieve template script", + "content": { + "text/plain": { + "schema": { + "type": "string" + } + } + } + } + } + } + }, + "/w/{workspace}/postgres_triggers/get_template_script/{id}": { + "get": { + "summary": "get template script", + "operationId": "getTemplateScript", + "tags": [ + "postgres_trigger" + ], + "parameters": [ + { + "$ref": "#/components/parameters/WorkspaceId" + }, + { + "$ref": "#/components/parameters/Id" + } + ], + "responses": { + "200": { + "description": "template script", + "content": { + "text/plain": { + "schema": { + "type": "string" + } + } + } + } + } + } + }, + "/w/{workspace}/postgres_triggers/slot/list/{path}": { + "get": { + "summary": "list postgres replication slot", + "operationId": "listPostgresReplicationSlot", + "tags": [ + "postgres_trigger" + ], + "parameters": [ + { + "$ref": "#/components/parameters/WorkspaceId" + }, + { + "$ref": "#/components/parameters/Path" + } + ], + "responses": { + "200": { + "description": "list postgres slot", + "content": { + "application/json": { + "schema": { + "type": "array", + "items": { + "$ref": "#/components/schemas/SlotList" + } + } + } + } + } + } + } + }, + "/w/{workspace}/postgres_triggers/slot/create/{path}": { + "post": { + "summary": "create replication slot for postgres", + "operationId": "createPostgresReplicationSlot", + "tags": [ + "postgres_trigger" + ], + "parameters": [ + { + "$ref": "#/components/parameters/WorkspaceId" + }, + { + "$ref": "#/components/parameters/Path" + } + ], + "requestBody": { + "description": "new slot for postgres", + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Slot" + } + } + } + }, + "responses": { + "201": { + "description": "slot created", + "content": { + "text/plain": { + "schema": { + "type": "string" + } + } + } + } + } + } + }, + "/w/{workspace}/postgres_triggers/slot/delete/{path}": { + "delete": { + "summary": "delete postgres replication slot", + "operationId": "deletePostgresReplicationSlot", + "tags": [ + "postgres_trigger" + ], + "parameters": [ + { + "$ref": "#/components/parameters/WorkspaceId" + }, + { + "$ref": "#/components/parameters/Path" + } + ], + "requestBody": { + "description": "replication slot of postgres", + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Slot" + } + } + } + }, + "responses": { + "200": { + "description": "postgres replication slot deleted", + "content": { + "text/plain": { + "schema": { + "type": "string" + } + } + } + } + } + } + }, + "/w/{workspace}/postgres_triggers/publication/list/{path}": { + "get": { + "summary": "list postgres publication", + "operationId": "listPostgresPublication", + "tags": [ + "postgres_trigger" + ], + "parameters": [ + { + "$ref": "#/components/parameters/WorkspaceId" + }, + { + "$ref": "#/components/parameters/Path" + } + ], + "responses": { + "200": { + "description": "database publication list", + "content": { + "application/json": { + "schema": { + "type": "array", + "items": { + "type": "string" + } + } + } + } + } + } + } + }, + "/w/{workspace}/postgres_triggers/publication/get/{publication}/{path}": { + "get": { + "summary": "get postgres publication", + "operationId": "getPostgresPublication", + "tags": [ + "postgres_trigger" + ], + "parameters": [ + { + "$ref": "#/components/parameters/WorkspaceId" + }, + { + "$ref": "#/components/parameters/Path" + }, + { + "$ref": "#/components/parameters/PublicationName" + } + ], + "responses": { + "200": { + "description": "postgres publication get", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/PublicationData" + } + } + } + } + } + } + }, + "/w/{workspace}/postgres_triggers/publication/create/{publication}/{path}": { + "post": { + "summary": "create publication for postgres", + "operationId": "createPostgresPublication", + "tags": [ + "postgres_trigger" + ], + "parameters": [ + { + "$ref": "#/components/parameters/WorkspaceId" + }, + { + "$ref": "#/components/parameters/Path" + }, + { + "$ref": "#/components/parameters/PublicationName" + } + ], + "requestBody": { + "description": "new publication for postgres", + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/PublicationData" + } + } + } + }, + "responses": { + "201": { + "description": "publication created", + "content": { + "text/plain": { + "schema": { + "type": "string" + } + } + } + } + } + } + }, + "/w/{workspace}/postgres_triggers/publication/update/{publication}/{path}": { + "post": { + "summary": "update publication for postgres", + "operationId": "updatePostgresPublication", + "tags": [ + "postgres_trigger" + ], + "parameters": [ + { + "$ref": "#/components/parameters/WorkspaceId" + }, + { + "$ref": "#/components/parameters/Path" + }, + { + "$ref": "#/components/parameters/PublicationName" + } + ], + "requestBody": { + "description": "update publication for postgres", + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/PublicationData" + } + } + } + }, + "responses": { + "201": { + "description": "publication updated", + "content": { + "text/plain": { + "schema": { + "type": "string" + } + } + } + } + } + } + }, + "/w/{workspace}/postgres_triggers/publication/delete/{publication}/{path}": { + "delete": { + "summary": "delete postgres publication", + "operationId": "deletePostgresPublication", + "tags": [ + "postgres_trigger" + ], + "parameters": [ + { + "$ref": "#/components/parameters/WorkspaceId" + }, + { + "$ref": "#/components/parameters/Path" + }, + { + "$ref": "#/components/parameters/PublicationName" + } + ], + "responses": { + "200": { + "description": "postgres publication deleted", + "content": { + "text/plain": { + "schema": { + "type": "string" + } + } + } + } + } + } + }, + "/w/{workspace}/postgres_triggers/create": { + "post": { + "summary": "create postgres trigger", + "operationId": "createPostgresTrigger", + "tags": [ + "postgres_trigger" + ], + "parameters": [ + { + "$ref": "#/components/parameters/WorkspaceId" + } + ], + "requestBody": { + "description": "new postgres trigger", + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/NewPostgresTrigger" + } + } + } + }, + "responses": { + "201": { + "description": "postgres trigger created", + "content": { + "text/plain": { + "schema": { + "type": "string" + } + } + } + } + } + } + }, + "/w/{workspace}/postgres_triggers/update/{path}": { + "post": { + "summary": "update postgres trigger", + "operationId": "updatePostgresTrigger", + "tags": [ + "postgres_trigger" + ], + "parameters": [ + { + "$ref": "#/components/parameters/WorkspaceId" + }, + { + "$ref": "#/components/parameters/Path" + } + ], + "requestBody": { + "description": "updated trigger", + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/EditPostgresTrigger" + } + } + } + }, + "responses": { + "200": { + "description": "postgres trigger updated", + "content": { + "text/plain": { + "schema": { + "type": "string" + } + } + } + } + } + } + }, + "/w/{workspace}/postgres_triggers/delete/{path}": { + "delete": { + "summary": "delete postgres trigger", + "operationId": "deletePostgresTrigger", + "tags": [ + "postgres_trigger" + ], + "parameters": [ + { + "$ref": "#/components/parameters/WorkspaceId" + }, + { + "$ref": "#/components/parameters/Path" + } + ], + "responses": { + "200": { + "description": "postgres trigger deleted", + "content": { + "text/plain": { + "schema": { + "type": "string" + } + } + } + } + } + } + }, + "/w/{workspace}/postgres_triggers/get/{path}": { + "get": { + "summary": "get postgres trigger", + "operationId": "getPostgresTrigger", + "tags": [ + "postgres_trigger" + ], + "parameters": [ + { + "$ref": "#/components/parameters/WorkspaceId" + }, + { + "$ref": "#/components/parameters/Path" + } + ], + "responses": { + "200": { + "description": "get postgres trigger", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/PostgresTrigger" + } + } + } + } + } + } + }, + "/w/{workspace}/postgres_triggers/list": { + "get": { + "summary": "list postgres triggers", + "operationId": "listPostgresTriggers", + "tags": [ + "postgres_trigger" + ], + "parameters": [ + { + "$ref": "#/components/parameters/WorkspaceId", + "required": true + }, + { + "$ref": "#/components/parameters/Page" + }, + { + "$ref": "#/components/parameters/PerPage" + }, + { + "name": "path", + "description": "filter by path", + "in": "query", + "schema": { + "type": "string" + } + }, + { + "name": "is_flow", + "in": "query", + "schema": { + "type": "boolean" + } + }, + { + "name": "path_start", + "in": "query", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "postgres trigger list", + "content": { + "application/json": { + "schema": { + "type": "array", + "items": { + "$ref": "#/components/schemas/PostgresTrigger" + } + } + } + } + } + } + } + }, + "/w/{workspace}/postgres_triggers/exists/{path}": { + "get": { + "summary": "does postgres trigger exists", + "operationId": "existsPostgresTrigger", + "tags": [ + "postgres_trigger" + ], + "parameters": [ + { + "$ref": "#/components/parameters/WorkspaceId" + }, + { + "$ref": "#/components/parameters/Path" + } + ], + "responses": { + "200": { + "description": "postgres trigger exists", + "content": { + "application/json": { + "schema": { + "type": "boolean" + } + } + } + } + } + } + }, + "/w/{workspace}/postgres_triggers/setenabled/{path}": { + "post": { + "summary": "set enabled postgres trigger", + "operationId": "setPostgresTriggerEnabled", + "tags": [ + "postgres_trigger" + ], + "parameters": [ + { + "$ref": "#/components/parameters/WorkspaceId" + }, + { + "$ref": "#/components/parameters/Path" + } + ], + "requestBody": { + "description": "updated postgres trigger enable", + "required": true, + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "enabled": { + "type": "boolean" + } + }, + "required": [ + "enabled" + ] + } + } + } + }, + "responses": { + "200": { + "description": "postgres trigger enabled set", + "content": { + "text/plain": { + "schema": { + "type": "string" + } + } + } + } + } + } + }, + "/w/{workspace}/postgres_triggers/test": { + "post": { + "summary": "test postgres connection", + "operationId": "testPostgresConnection", + "tags": [ + "postgres_trigger" + ], + "parameters": [ + { + "$ref": "#/components/parameters/WorkspaceId" + } + ], + "requestBody": { + "description": "test postgres connection", + "required": true, + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "database": { + "type": "string" + } + }, + "required": [ + "database" + ] + } + } + } + }, + "responses": { + "200": { + "description": "successfuly connected to postgres", + "content": { + "text/plain": { + "schema": { + "type": "string" + } + } + } + } + } + } + }, "/groups/list": { "get": { "summary": "list instance groups", @@ -12989,6 +17483,35 @@ } } }, + "/w/{workspace}/folders/exists/{name}": { + "get": { + "summary": "exists folder", + "operationId": "existsFolder", + "tags": [ + "folder" + ], + "parameters": [ + { + "$ref": "#/components/parameters/WorkspaceId" + }, + { + "$ref": "#/components/parameters/Name" + } + ], + "responses": { + "200": { + "description": "folder exists", + "content": { + "application/json": { + "schema": { + "type": "boolean" + } + } + } + } + } + } + }, "/w/{workspace}/folders/getusage/{name}": { "get": { "summary": "get folder usage", @@ -13343,7 +17866,9 @@ "description": "a config", "content": { "application/json": { - "schema": {} + "schema": { + "$ref": "#/components/schemas/Configs" + } } } } @@ -13467,6 +17992,81 @@ } } }, + "/configs/list_available_python_versions": { + "get": { + "summary": "Get currently available python versions provided by UV.", + "operationId": "listAvailablePythonVersions", + "tags": [ + "config" + ], + "responses": { + "200": { + "description": "List of python versions", + "content": { + "application/json": { + "schema": { + "type": "array", + "items": { + "type": "string" + } + } + } + } + } + } + } + }, + "/agent_workers/create_agent_token": { + "post": { + "summary": "create agent token", + "operationId": "createAgentToken", + "tags": [ + "agent_workers" + ], + "requestBody": { + "description": "agent token", + "required": true, + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "worker_group": { + "type": "string" + }, + "tags": { + "type": "array", + "items": { + "type": "string" + } + }, + "exp": { + "type": "integer" + } + }, + "required": [ + "worker_group", + "tags", + "exp" + ] + } + } + } + }, + "responses": { + "200": { + "description": "agent token created", + "content": { + "application/json": { + "schema": { + "type": "string" + } + } + } + } + } + } + }, "/w/{workspace}/acls/get/{kind}/{path}": { "get": { "summary": "get granular acls", @@ -13498,7 +18098,13 @@ "app", "raw_app", "http_trigger", - "websocket_trigger" + "websocket_trigger", + "kafka_trigger", + "nats_trigger", + "postgres_trigger", + "mqtt_trigger", + "gcp_trigger", + "sqs_trigger" ] } } @@ -13551,7 +18157,13 @@ "app", "raw_app", "http_trigger", - "websocket_trigger" + "websocket_trigger", + "kafka_trigger", + "nats_trigger", + "postgres_trigger", + "mqtt_trigger", + "gcp_trigger", + "sqs_trigger" ] } } @@ -13623,7 +18235,13 @@ "app", "raw_app", "http_trigger", - "websocket_trigger" + "websocket_trigger", + "kafka_trigger", + "nats_trigger", + "postgres_trigger", + "mqtt_trigger", + "gcp_trigger", + "sqs_trigger" ] } } @@ -13661,32 +18279,66 @@ } } }, - "/w/{workspace}/capture_u/{path}": { + "/w/{workspace}/capture/set_config": { "post": { - "summary": "update flow preview capture", - "operationId": "updateCapture", + "summary": "set capture config", + "operationId": "setCaptureConfig", "tags": [ "capture" ], "parameters": [ { "$ref": "#/components/parameters/WorkspaceId" - }, - { - "$ref": "#/components/parameters/Path" } ], + "requestBody": { + "description": "capture config", + "required": true, + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "trigger_kind": { + "$ref": "#/components/schemas/CaptureTriggerKind" + }, + "path": { + "type": "string" + }, + "is_flow": { + "type": "boolean" + }, + "trigger_config": { + "type": "object" + } + }, + "required": [ + "trigger_kind", + "path", + "is_flow" + ] + } + } + } + }, "responses": { - "204": { - "description": "flow preview captured" + "200": { + "description": "capture config set", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } } } } }, - "/w/{workspace}/capture/{path}": { - "put": { - "summary": "create flow preview capture", - "operationId": "createCapture", + "/w/{workspace}/capture/ping_config/{trigger_kind}/{runnable_kind}/{path}": { + "post": { + "summary": "ping capture config", + "operationId": "pingCaptureConfig", "tags": [ "capture" ], @@ -13694,18 +18346,162 @@ { "$ref": "#/components/parameters/WorkspaceId" }, + { + "name": "trigger_kind", + "in": "path", + "required": true, + "schema": { + "$ref": "#/components/schemas/CaptureTriggerKind" + } + }, + { + "$ref": "#/components/parameters/RunnableKind" + }, { "$ref": "#/components/parameters/Path" } ], "responses": { - "201": { - "description": "flow preview capture created" + "200": { + "description": "capture config pinged" } } - }, + } + }, + "/w/{workspace}/capture/get_configs/{runnable_kind}/{path}": { "get": { - "summary": "get flow preview capture", + "summary": "get capture configs for a script or flow", + "operationId": "getCaptureConfigs", + "tags": [ + "capture" + ], + "parameters": [ + { + "$ref": "#/components/parameters/WorkspaceId" + }, + { + "$ref": "#/components/parameters/RunnableKind" + }, + { + "$ref": "#/components/parameters/Path" + } + ], + "responses": { + "200": { + "description": "capture configs for a script or flow", + "content": { + "application/json": { + "schema": { + "type": "array", + "items": { + "$ref": "#/components/schemas/CaptureConfig" + } + } + } + } + } + } + } + }, + "/w/{workspace}/capture/list/{runnable_kind}/{path}": { + "get": { + "summary": "list captures for a script or flow", + "operationId": "listCaptures", + "tags": [ + "capture" + ], + "parameters": [ + { + "$ref": "#/components/parameters/WorkspaceId" + }, + { + "$ref": "#/components/parameters/RunnableKind" + }, + { + "$ref": "#/components/parameters/Path" + }, + { + "name": "trigger_kind", + "in": "query", + "schema": { + "$ref": "#/components/schemas/CaptureTriggerKind" + } + }, + { + "$ref": "#/components/parameters/Page" + }, + { + "$ref": "#/components/parameters/PerPage" + } + ], + "responses": { + "200": { + "description": "list of captures for a script or flow", + "content": { + "application/json": { + "schema": { + "type": "array", + "items": { + "$ref": "#/components/schemas/Capture" + } + } + } + } + } + } + } + }, + "/w/{workspace}/capture/move/{runnable_kind}/{path}": { + "post": { + "summary": "move captures and configs for a script or flow", + "operationId": "moveCapturesAndConfigs", + "tags": [ + "capture" + ], + "parameters": [ + { + "$ref": "#/components/parameters/WorkspaceId" + }, + { + "$ref": "#/components/parameters/RunnableKind" + }, + { + "$ref": "#/components/parameters/Path" + } + ], + "requestBody": { + "description": "move captures and configs to a new path", + "required": true, + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "new_path": { + "type": "string" + } + } + } + } + } + }, + "responses": { + "200": { + "description": "captures and configs moved", + "content": { + "text/plain": { + "schema": { + "type": "string" + } + } + } + } + } + } + }, + "/w/{workspace}/capture/{id}": { + "get": { + "summary": "get a capture", "operationId": "getCapture", "tags": [ "capture" @@ -13715,20 +18511,49 @@ "$ref": "#/components/parameters/WorkspaceId" }, { - "$ref": "#/components/parameters/Path" + "name": "id", + "in": "path", + "required": true, + "schema": { + "type": "integer" + } } ], "responses": { "200": { - "description": "captured flow preview", + "description": "capture", "content": { "application/json": { - "schema": {} + "schema": { + "$ref": "#/components/schemas/Capture" + } } } + } + } + }, + "delete": { + "summary": "delete a capture", + "operationId": "deleteCapture", + "tags": [ + "capture" + ], + "parameters": [ + { + "$ref": "#/components/parameters/WorkspaceId" }, - "404": { - "description": "capture does not exist for this flow" + { + "name": "id", + "in": "path", + "required": true, + "schema": { + "type": "integer" + } + } + ], + "responses": { + "200": { + "description": "capture deleted" } } } @@ -13839,6 +18664,16 @@ }, { "$ref": "#/components/parameters/PerPage" + }, + { + "$ref": "#/components/parameters/ArgsFilter" + }, + { + "name": "include_preview", + "in": "query", + "schema": { + "type": "boolean" + } } ], "responses": { @@ -14142,6 +18977,9 @@ "properties": { "connection_settings_str": { "type": "string" + }, + "azure_container_path": { + "type": "string" } }, "required": [ @@ -14994,7 +19832,7 @@ }, "/w/{workspace}/job_helpers/download_s3_file": { "get": { - "summary": "Download file to S3 bucket", + "summary": "Download file from S3 bucket", "operationId": "fileDownload", "tags": [ "helpers" @@ -15508,6 +20346,9 @@ { "$ref": "#/components/parameters/ResultFilter" }, + { + "$ref": "#/components/parameters/AllowWildcards" + }, { "$ref": "#/components/parameters/Page" }, @@ -15595,6 +20436,14 @@ "schema": { "type": "string" } + }, + { + "name": "pagination_offset", + "in": "query", + "required": false, + "schema": { + "type": "integer" + } } ], "responses": { @@ -15609,12 +20458,7 @@ "description": "a list of the terms that couldn't be parsed (and thus ignored)", "type": "array", "items": { - "type": "object", - "properties": { - "dancer": { - "type": "string" - } - } + "type": "string" } }, "hits": { @@ -15623,6 +20467,25 @@ "items": { "$ref": "#/components/schemas/JobSearchHit" } + }, + "hit_count": { + "description": "how many jobs matched in total", + "type": "number" + }, + "index_metadata": { + "description": "Metadata about the index current state", + "type": "object", + "properties": { + "indexed_until": { + "description": "Datetime of the most recently indexed job", + "type": "string", + "format": "date-time" + }, + "lost_lock_ownership": { + "description": "Is the current indexer service being replaced", + "type": "boolean" + } + } } } } @@ -15737,14 +20600,6 @@ "type": "string" } }, - { - "name": "hosts", - "in": "query", - "required": true, - "schema": { - "type": "string" - } - }, { "name": "min_ts", "in": "query", @@ -15790,6 +20645,41 @@ } } } + }, + "/srch/index/delete/{idx_name}": { + "delete": { + "summary": "Restart container and delete the index to recreate it.", + "operationId": "clearIndex", + "tags": [ + "indexSearch" + ], + "parameters": [ + { + "name": "idx_name", + "in": "path", + "required": true, + "schema": { + "type": "string", + "enum": [ + "JobIndex", + "ServiceLogIndex" + ] + } + } + ], + "responses": { + "200": { + "description": "idx to be deleted and container restarting", + "content": { + "text/plain": { + "schema": { + "type": "string" + } + } + } + } + } + } } }, "components": { @@ -15805,6 +20695,14 @@ } }, "parameters": { + "Id": { + "name": "id", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, "Key": { "name": "key", "in": "path", @@ -15821,6 +20719,14 @@ "type": "string" } }, + "PublicationName": { + "name": "publication", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, "VersionId": { "name": "version", "in": "path", @@ -15886,6 +20792,14 @@ "type": "string" } }, + "CustomPath": { + "name": "custom_path", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, "PathId": { "name": "id", "in": "path", @@ -15950,6 +20864,14 @@ "type": "string" } }, + "Worker": { + "name": "worker", + "description": "worker this job was ran on", + "in": "query", + "schema": { + "type": "string" + } + }, "ParentJob": { "name": "parent_job", "description": "The parent job that is at the origin and responsible for the execution of this script if any", @@ -16144,6 +21066,14 @@ "type": "boolean" } }, + "AllowWildcards": { + "name": "allow_wildcards", + "description": "allow wildcards (*) in the filter of label, tag, worker", + "in": "query", + "schema": { + "type": "boolean" + } + }, "ArgsFilter": { "name": "args", "description": "filter on jobs containing those args as a json subset (@> in postgres)", @@ -16259,24 +21189,135 @@ "schema": { "type": "string" } + }, + "RunnableKind": { + "name": "runnable_kind", + "in": "path", + "required": true, + "schema": { + "type": "string", + "enum": [ + "script", + "flow" + ] + } } }, "schemas": { - "AiResource": { + "InputTransform": { + "allOf": [ + { + "$ref": "#/components/schemas/schemas-InputTransform" + } + ] + }, + "AIProvider": { + "type": "string", + "enum": [ + "openai", + "azure_openai", + "anthropic", + "mistral", + "deepseek", + "googleai", + "groq", + "openrouter", + "togetherai", + "customai" + ] + }, + "AIProviderModel": { "type": "object", "properties": { - "path": { + "model": { "type": "string" }, "provider": { - "type": "string" + "$ref": "#/components/schemas/AIProvider" } }, "required": [ - "path", + "model", "provider" ] }, + "AIProviderConfig": { + "type": "object", + "properties": { + "resource_path": { + "type": "string" + }, + "models": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "resource_path", + "models" + ] + }, + "AIConfig": { + "type": "object", + "properties": { + "providers": { + "type": "object", + "additionalProperties": { + "$ref": "#/components/schemas/AIProviderConfig" + } + }, + "default_model": { + "$ref": "#/components/schemas/AIProviderModel" + }, + "code_completion_model": { + "$ref": "#/components/schemas/AIProviderModel" + } + } + }, + "Alert": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "tags_to_monitor": { + "type": "array", + "items": { + "type": "string" + } + }, + "jobs_num_threshold": { + "type": "integer" + }, + "alert_cooldown_seconds": { + "type": "integer" + }, + "alert_time_threshold_seconds": { + "type": "integer" + } + }, + "required": [ + "name", + "tags_to_monitor", + "jobs_num_threshold", + "alert_cooldown_seconds", + "alert_time_threshold_seconds" + ] + }, + "Configs": { + "type": "object", + "nullable": true, + "properties": { + "alerts": { + "type": "array", + "items": { + "$ref": "#/components/schemas/Alert" + } + } + } + }, "Script": { "type": "object", "properties": { @@ -16337,25 +21378,7 @@ "type": "string" }, "language": { - "type": "string", - "enum": [ - "python3", - "deno", - "go", - "bash", - "powershell", - "postgresql", - "mysql", - "bigquery", - "snowflake", - "mssql", - "graphql", - "nativets", - "bun", - "php", - "rust", - "ansible" - ] + "$ref": "#/components/schemas/ScriptLang" }, "kind": { "type": "string", @@ -16364,7 +21387,8 @@ "failure", "trigger", "command", - "approval" + "approval", + "preprocessor" ] }, "starred": { @@ -16426,6 +21450,9 @@ }, "has_preprocessor": { "type": "boolean" + }, + "on_behalf_of_email": { + "type": "string" } }, "required": [ @@ -16475,25 +21502,7 @@ "type": "string" }, "language": { - "type": "string", - "enum": [ - "python3", - "deno", - "go", - "bash", - "powershell", - "postgresql", - "mysql", - "bigquery", - "snowflake", - "mssql", - "graphql", - "nativets", - "bun", - "php", - "rust", - "ansible" - ] + "$ref": "#/components/schemas/ScriptLang" }, "kind": { "type": "string", @@ -16502,7 +21511,8 @@ "failure", "trigger", "command", - "approval" + "approval", + "preprocessor" ] }, "tag": { @@ -16561,6 +21571,9 @@ }, "has_preprocessor": { "type": "boolean" + }, + "on_behalf_of_email": { + "type": "string" } }, "required": [ @@ -16758,7 +21771,10 @@ "script_hub", "identity", "deploymentcallback", - "singlescriptflow" + "singlescriptflow", + "flowscript", + "flownode", + "appscript" ] }, "schedule_path": { @@ -16766,7 +21782,7 @@ }, "permissioned_as": { "type": "string", - "description": "The user (u/userfoo) or group (g/groupfoo) whom \nthe execution of this script will be permissioned_as and by extension its DT_TOKEN.\n" + "description": "The user (u/userfoo) or group (g/groupfoo) whom\nthe execution of this script will be permissioned_as and by extension its DT_TOKEN.\n" }, "flow_status": { "$ref": "#/components/schemas/FlowStatus" @@ -16778,25 +21794,7 @@ "type": "boolean" }, "language": { - "type": "string", - "enum": [ - "python3", - "deno", - "go", - "bash", - "powershell", - "postgresql", - "mysql", - "bigquery", - "snowflake", - "mssql", - "graphql", - "nativets", - "bun", - "php", - "rust", - "ansible" - ] + "$ref": "#/components/schemas/ScriptLang" }, "email": { "type": "string" @@ -16821,6 +21819,12 @@ }, "suspend": { "type": "number" + }, + "preprocessed": { + "type": "boolean" + }, + "worker": { + "type": "string" } }, "required": [ @@ -16907,7 +21911,10 @@ "script_hub", "identity", "deploymentcallback", - "singlescriptflow" + "singlescriptflow", + "flowscript", + "flownode", + "appscript" ] }, "schedule_path": { @@ -16915,7 +21922,7 @@ }, "permissioned_as": { "type": "string", - "description": "The user (u/userfoo) or group (g/groupfoo) whom \nthe execution of this script will be permissioned_as and by extension its DT_TOKEN.\n" + "description": "The user (u/userfoo) or group (g/groupfoo) whom\nthe execution of this script will be permissioned_as and by extension its DT_TOKEN.\n" }, "flow_status": { "$ref": "#/components/schemas/FlowStatus" @@ -16927,25 +21934,7 @@ "type": "boolean" }, "language": { - "type": "string", - "enum": [ - "python3", - "deno", - "go", - "bash", - "powershell", - "postgresql", - "mysql", - "bigquery", - "snowflake", - "mssql", - "graphql", - "nativets", - "bun", - "php", - "rust", - "ansible" - ] + "$ref": "#/components/schemas/ScriptLang" }, "is_skipped": { "type": "boolean" @@ -16976,6 +21965,12 @@ }, "aggregate_wait_time_ms": { "type": "number" + }, + "preprocessed": { + "type": "boolean" + }, + "worker": { + "type": "string" } }, "required": [ @@ -17065,6 +22060,9 @@ "is_admin": { "type": "boolean" }, + "name": { + "type": "string" + }, "is_super_admin": { "type": "boolean" }, @@ -17670,6 +22668,31 @@ "has_preprocessor" ] }, + "ScriptLang": { + "type": "string", + "enum": [ + "python3", + "deno", + "go", + "bash", + "powershell", + "postgresql", + "mysql", + "bigquery", + "snowflake", + "mssql", + "oracledb", + "graphql", + "nativets", + "bun", + "php", + "rust", + "ansible", + "csharp", + "nu", + "java" + ] + }, "Preview": { "type": "object", "properties": { @@ -17679,29 +22702,14 @@ "path": { "type": "string" }, + "script_hash": { + "type": "string" + }, "args": { "$ref": "#/components/schemas/ScriptArgs" }, "language": { - "type": "string", - "enum": [ - "python3", - "deno", - "go", - "bash", - "powershell", - "postgresql", - "mysql", - "bigquery", - "snowflake", - "mssql", - "graphql", - "nativets", - "bun", - "php", - "rust", - "ansible" - ] + "$ref": "#/components/schemas/ScriptLang" }, "tag": { "type": "string" @@ -18004,6 +23012,9 @@ "summary": { "type": "string" }, + "description": { + "type": "string" + }, "no_flow_overlap": { "type": "boolean" }, @@ -18013,6 +23024,9 @@ "paused_until": { "type": "string", "format": "date-time" + }, + "cron_version": { + "type": "string" } }, "required": [ @@ -18125,12 +23139,18 @@ "summary": { "type": "string" }, + "description": { + "type": "string" + }, "tag": { "type": "string" }, "paused_until": { "type": "string", "format": "date-time" + }, + "cron_version": { + "type": "string" } }, "required": [ @@ -18193,12 +23213,18 @@ "summary": { "type": "string" }, + "description": { + "type": "string" + }, "tag": { "type": "string" }, "paused_until": { "type": "string", "format": "date-time" + }, + "cron_version": { + "type": "string" } }, "required": [ @@ -18209,12 +23235,27 @@ "args" ] }, - "HttpTrigger": { + "TriggerExtraProperty": { "type": "object", "properties": { "path": { "type": "string" }, + "script_path": { + "type": "string" + }, + "email": { + "type": "string" + }, + "extra_perms": { + "type": "object", + "additionalProperties": { + "type": "boolean" + } + }, + "workspace_id": { + "type": "string" + }, "edited_by": { "type": "string" }, @@ -18222,9 +23263,40 @@ "type": "string", "format": "date-time" }, - "script_path": { - "type": "string" - }, + "is_flow": { + "type": "boolean" + } + }, + "required": [ + "path", + "script_path", + "email", + "extra_perms", + "workspace_id", + "edited_by", + "edited_at", + "is_flow" + ] + }, + "AuthenticationMethod": { + "type": "string", + "enum": [ + "none", + "windmill", + "api_key", + "basic_http", + "custom_script", + "signature" + ] + }, + "HttpTrigger": { + "allOf": [ + { + "$ref": "#/components/schemas/TriggerExtraProperty" + } + ], + "type": "object", + "properties": { "route_path": { "type": "string" }, @@ -18245,21 +23317,6 @@ "s3" ] }, - "is_flow": { - "type": "boolean" - }, - "extra_perms": { - "type": "object", - "additionalProperties": { - "type": "boolean" - } - }, - "email": { - "type": "string" - }, - "workspace_id": { - "type": "string" - }, "http_method": { "type": "string", "enum": [ @@ -18270,26 +23327,37 @@ "patch" ] }, + "authentication_resource_path": { + "type": "string" + }, "is_async": { "type": "boolean" }, - "requires_auth": { + "authentication_method": { + "$ref": "#/components/schemas/AuthenticationMethod" + }, + "is_static_website": { + "type": "boolean" + }, + "workspaced_route": { + "type": "boolean" + }, + "wrap_body": { + "type": "boolean" + }, + "raw_string": { "type": "boolean" } }, "required": [ - "path", - "edited_by", - "edited_at", - "script_path", "route_path", - "extra_perms", - "is_flow", - "email", - "workspace_id", "is_async", - "requires_auth", - "http_method" + "authentication_method", + "http_method", + "is_static_website", + "workspaced_route", + "wrap_body", + "raw_string" ] }, "NewHttpTrigger": { @@ -18304,6 +23372,9 @@ "route_path": { "type": "string" }, + "workspaced_route": { + "type": "boolean" + }, "static_asset_config": { "type": "object", "properties": { @@ -18334,10 +23405,22 @@ "patch" ] }, + "authentication_resource_path": { + "type": "string" + }, "is_async": { "type": "boolean" }, - "requires_auth": { + "authentication_method": { + "$ref": "#/components/schemas/AuthenticationMethod" + }, + "is_static_website": { + "type": "boolean" + }, + "wrap_body": { + "type": "boolean" + }, + "raw_string": { "type": "boolean" } }, @@ -18347,8 +23430,9 @@ "route_path", "is_flow", "is_async", - "requires_auth", - "http_method" + "authentication_method", + "http_method", + "is_static_website" ] }, "EditHttpTrigger": { @@ -18363,6 +23447,9 @@ "route_path": { "type": "string" }, + "workspaced_route": { + "type": "boolean" + }, "static_asset_config": { "type": "object", "properties": { @@ -18380,6 +23467,9 @@ "s3" ] }, + "authentication_resource_path": { + "type": "string" + }, "is_flow": { "type": "boolean" }, @@ -18396,7 +23486,16 @@ "is_async": { "type": "boolean" }, - "requires_auth": { + "authentication_method": { + "$ref": "#/components/schemas/AuthenticationMethod" + }, + "is_static_website": { + "type": "boolean" + }, + "wrap_body": { + "type": "boolean" + }, + "raw_string": { "type": "boolean" } }, @@ -18406,8 +23505,9 @@ "is_flow", "kind", "is_async", - "requires_auth", - "http_method" + "authentication_method", + "http_method", + "is_static_website" ] }, "TriggersCount": { @@ -18435,43 +23535,38 @@ }, "websocket_count": { "type": "number" + }, + "postgres_count": { + "type": "number" + }, + "kafka_count": { + "type": "number" + }, + "nats_count": { + "type": "number" + }, + "mqtt_count": { + "type": "number" + }, + "gcp_count": { + "type": "number" + }, + "sqs_count": { + "type": "number" } } }, "WebsocketTrigger": { + "allOf": [ + { + "$ref": "#/components/schemas/TriggerExtraProperty" + } + ], "type": "object", "properties": { - "path": { - "type": "string" - }, - "edited_by": { - "type": "string" - }, - "edited_at": { - "type": "string", - "format": "date-time" - }, - "script_path": { - "type": "string" - }, "url": { "type": "string" }, - "is_flow": { - "type": "boolean" - }, - "extra_perms": { - "type": "object", - "additionalProperties": { - "type": "boolean" - } - }, - "email": { - "type": "string" - }, - "workspace_id": { - "type": "string" - }, "server_id": { "type": "string" }, @@ -18509,22 +23604,16 @@ }, "url_runnable_args": { "$ref": "#/components/schemas/ScriptArgs" + }, + "can_return_message": { + "type": "boolean" } }, "required": [ - "path", - "edited_by", - "edited_at", - "script_path", "url", - "extra_perms", - "is_flow", - "email", - "workspace_id", "enabled", "filters", - "initial_messages", - "url_runnable_args" + "can_return_message" ] }, "NewWebsocketTrigger": { @@ -18569,6 +23658,9 @@ }, "url_runnable_args": { "$ref": "#/components/schemas/ScriptArgs" + }, + "can_return_message": { + "type": "boolean" } }, "required": [ @@ -18577,8 +23669,7 @@ "url", "is_flow", "filters", - "initial_messages", - "url_runnable_args" + "can_return_message" ] }, "EditWebsocketTrigger": { @@ -18620,6 +23711,9 @@ }, "url_runnable_args": { "$ref": "#/components/schemas/ScriptArgs" + }, + "can_return_message": { + "type": "boolean" } }, "required": [ @@ -18628,8 +23722,7 @@ "url", "is_flow", "filters", - "initial_messages", - "url_runnable_args" + "can_return_message" ] }, "WebsocketTriggerInitialMessage": { @@ -18674,6 +23767,936 @@ } ] }, + "MqttQoS": { + "type": "string", + "enum": [ + "qos0", + "qos1", + "qos2" + ] + }, + "MqttV3Config": { + "type": "object", + "properties": { + "clean_session": { + "type": "boolean" + } + } + }, + "MqttV5Config": { + "type": "object", + "properties": { + "clean_start": { + "type": "boolean" + }, + "topic_alias": { + "type": "number" + }, + "session_expiry_interval": { + "type": "number" + } + } + }, + "MqttSubscribeTopic": { + "type": "object", + "properties": { + "qos": { + "$ref": "#/components/schemas/MqttQoS" + }, + "topic": { + "type": "string" + } + }, + "required": [ + "qos", + "topic" + ] + }, + "MqttClientVersion": { + "type": "string", + "enum": [ + "v3", + "v5" + ] + }, + "MqttTrigger": { + "allOf": [ + { + "$ref": "#/components/schemas/TriggerExtraProperty" + } + ], + "type": "object", + "properties": { + "mqtt_resource_path": { + "type": "string" + }, + "subscribe_topics": { + "type": "array", + "items": { + "$ref": "#/components/schemas/MqttSubscribeTopic" + } + }, + "v3_config": { + "$ref": "#/components/schemas/MqttV3Config" + }, + "v5_config": { + "$ref": "#/components/schemas/MqttV5Config" + }, + "client_id": { + "type": "string" + }, + "client_version": { + "$ref": "#/components/schemas/MqttClientVersion" + }, + "server_id": { + "type": "string" + }, + "last_server_ping": { + "type": "string", + "format": "date-time" + }, + "error": { + "type": "string" + }, + "enabled": { + "type": "boolean" + } + }, + "required": [ + "enabled", + "subscribe_topics", + "mqtt_resource_path" + ] + }, + "NewMqttTrigger": { + "type": "object", + "properties": { + "mqtt_resource_path": { + "type": "string" + }, + "subscribe_topics": { + "type": "array", + "items": { + "$ref": "#/components/schemas/MqttSubscribeTopic" + } + }, + "client_id": { + "type": "string" + }, + "v3_config": { + "$ref": "#/components/schemas/MqttV3Config" + }, + "v5_config": { + "$ref": "#/components/schemas/MqttV5Config" + }, + "client_version": { + "$ref": "#/components/schemas/MqttClientVersion" + }, + "path": { + "type": "string" + }, + "script_path": { + "type": "string" + }, + "is_flow": { + "type": "boolean" + }, + "enabled": { + "type": "boolean" + } + }, + "required": [ + "path", + "script_path", + "is_flow", + "subscribe_topics", + "mqtt_resource_path" + ] + }, + "EditMqttTrigger": { + "type": "object", + "properties": { + "mqtt_resource_path": { + "type": "string" + }, + "subscribe_topics": { + "type": "array", + "items": { + "$ref": "#/components/schemas/MqttSubscribeTopic" + } + }, + "client_id": { + "type": "string" + }, + "v3_config": { + "$ref": "#/components/schemas/MqttV3Config" + }, + "v5_config": { + "$ref": "#/components/schemas/MqttV5Config" + }, + "client_version": { + "$ref": "#/components/schemas/MqttClientVersion" + }, + "path": { + "type": "string" + }, + "script_path": { + "type": "string" + }, + "is_flow": { + "type": "boolean" + }, + "enabled": { + "type": "boolean" + } + }, + "required": [ + "path", + "script_path", + "is_flow", + "enabled", + "subscribe_topics", + "mqtt_resource_path" + ] + }, + "DeliveryType": { + "type": "string", + "enum": [ + "push", + "pull" + ] + }, + "PushConfig": { + "type": "object", + "properties": { + "audience": { + "type": "string" + }, + "authenticate": { + "type": "boolean" + } + }, + "required": [ + "authenticate", + "base_endpoint" + ] + }, + "GcpTrigger": { + "allOf": [ + { + "$ref": "#/components/schemas/TriggerExtraProperty" + } + ], + "type": "object", + "properties": { + "gcp_resource_path": { + "type": "string" + }, + "topic_id": { + "type": "string" + }, + "subscription_id": { + "type": "string" + }, + "server_id": { + "type": "string" + }, + "delivery_type": { + "$ref": "#/components/schemas/DeliveryType" + }, + "delivery_config": { + "$ref": "#/components/schemas/PushConfig" + }, + "subscription_mode": { + "$ref": "#/components/schemas/SubscriptionMode" + }, + "last_server_ping": { + "type": "string", + "format": "date-time" + }, + "error": { + "type": "string" + }, + "enabled": { + "type": "boolean" + } + }, + "required": [ + "gcp_resource_path", + "topic_id", + "subscription_id", + "enabled", + "delivery_type", + "subscription_mode" + ] + }, + "SubscriptionMode": { + "type": "string", + "enum": [ + "existing", + "create_update" + ], + "description": "The mode of subscription. 'existing' means using an existing GCP subscription, while 'create_update' involves creating or updating a new subscription." + }, + "GcpTriggerData": { + "type": "object", + "properties": { + "gcp_resource_path": { + "type": "string" + }, + "subscription_mode": { + "$ref": "#/components/schemas/SubscriptionMode" + }, + "topic_id": { + "type": "string" + }, + "subscription_id": { + "type": "string" + }, + "base_endpoint": { + "type": "string" + }, + "delivery_type": { + "$ref": "#/components/schemas/DeliveryType" + }, + "delivery_config": { + "$ref": "#/components/schemas/PushConfig" + }, + "path": { + "type": "string" + }, + "script_path": { + "type": "string" + }, + "is_flow": { + "type": "boolean" + }, + "enabled": { + "type": "boolean" + } + }, + "required": [ + "path", + "script_path", + "is_flow", + "gcp_resource_path", + "topic_id", + "subscription_mode" + ] + }, + "GetAllTopicSubscription": { + "type": "object", + "properties": { + "topic_id": { + "type": "string" + } + }, + "required": [ + "topic_id" + ] + }, + "DeleteGcpSubscription": { + "type": "object", + "properties": { + "subscription_id": { + "type": "string" + } + }, + "required": [ + "subscription_id" + ] + }, + "AwsAuthResourceType": { + "type": "string", + "enum": [ + "oidc", + "credentials" + ] + }, + "SqsTrigger": { + "allOf": [ + { + "$ref": "#/components/schemas/TriggerExtraProperty" + } + ], + "type": "object", + "properties": { + "queue_url": { + "type": "string" + }, + "aws_auth_resource_type": { + "$ref": "#/components/schemas/AwsAuthResourceType" + }, + "aws_resource_path": { + "type": "string" + }, + "message_attributes": { + "type": "array", + "items": { + "type": "string" + } + }, + "server_id": { + "type": "string" + }, + "last_server_ping": { + "type": "string", + "format": "date-time" + }, + "error": { + "type": "string" + }, + "enabled": { + "type": "boolean" + } + }, + "required": [ + "queue_url", + "aws_resource_path", + "enabled", + "aws_auth_resource_type" + ] + }, + "NewSqsTrigger": { + "type": "object", + "properties": { + "queue_url": { + "type": "string" + }, + "aws_auth_resource_type": { + "$ref": "#/components/schemas/AwsAuthResourceType" + }, + "aws_resource_path": { + "type": "string" + }, + "message_attributes": { + "type": "array", + "items": { + "type": "string" + } + }, + "path": { + "type": "string" + }, + "script_path": { + "type": "string" + }, + "is_flow": { + "type": "boolean" + }, + "enabled": { + "type": "boolean" + } + }, + "required": [ + "queue_url", + "aws_resource_path", + "path", + "script_path", + "is_flow", + "aws_auth_resource_type" + ] + }, + "EditSqsTrigger": { + "type": "object", + "properties": { + "queue_url": { + "type": "string" + }, + "aws_auth_resource_type": { + "$ref": "#/components/schemas/AwsAuthResourceType" + }, + "aws_resource_path": { + "type": "string" + }, + "message_attributes": { + "type": "array", + "items": { + "type": "string" + } + }, + "path": { + "type": "string" + }, + "script_path": { + "type": "string" + }, + "is_flow": { + "type": "boolean" + }, + "enabled": { + "type": "boolean" + } + }, + "required": [ + "queue_url", + "aws_resource_path", + "path", + "script_path", + "is_flow", + "enabled", + "aws_auth_resource_type" + ] + }, + "Slot": { + "type": "object", + "properties": { + "name": { + "type": "string" + } + } + }, + "SlotList": { + "type": "object", + "properties": { + "slot_name": { + "type": "string" + }, + "active": { + "type": "boolean" + } + } + }, + "PublicationData": { + "type": "object", + "properties": { + "table_to_track": { + "type": "array", + "items": { + "$ref": "#/components/schemas/Relations" + } + }, + "transaction_to_track": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "transaction_to_track" + ] + }, + "TableToTrack": { + "type": "array", + "items": { + "type": "object", + "properties": { + "table_name": { + "type": "string" + }, + "columns_name": { + "type": "array", + "items": { + "type": "string" + } + }, + "where_clause": { + "type": "string" + } + }, + "required": [ + "table_name" + ] + } + }, + "Relations": { + "type": "object", + "properties": { + "schema_name": { + "type": "string" + }, + "table_to_track": { + "$ref": "#/components/schemas/TableToTrack" + } + }, + "required": [ + "schema_name", + "table_to_track" + ] + }, + "Language": { + "type": "string", + "enum": [ + "Typescript" + ] + }, + "TemplateScript": { + "type": "object", + "properties": { + "postgres_resource_path": { + "type": "string" + }, + "relations": { + "type": "array", + "items": { + "$ref": "#/components/schemas/Relations" + } + }, + "language": { + "$ref": "#/components/schemas/Language" + } + }, + "required": [ + "postgres_resource_path", + "relations", + "language" + ] + }, + "PostgresTrigger": { + "allOf": [ + { + "$ref": "#/components/schemas/TriggerExtraProperty" + } + ], + "type": "object", + "properties": { + "enabled": { + "type": "boolean" + }, + "postgres_resource_path": { + "type": "string" + }, + "publication_name": { + "type": "string" + }, + "server_id": { + "type": "string" + }, + "replication_slot_name": { + "type": "string" + }, + "error": { + "type": "string" + }, + "last_server_ping": { + "type": "string", + "format": "date-time" + } + }, + "required": [ + "enabled", + "postgres_resource_path", + "replication_slot_name", + "publication_name" + ] + }, + "NewPostgresTrigger": { + "type": "object", + "properties": { + "replication_slot_name": { + "type": "string" + }, + "publication_name": { + "type": "string" + }, + "path": { + "type": "string" + }, + "script_path": { + "type": "string" + }, + "is_flow": { + "type": "boolean" + }, + "enabled": { + "type": "boolean" + }, + "postgres_resource_path": { + "type": "string" + }, + "publication": { + "$ref": "#/components/schemas/PublicationData" + } + }, + "required": [ + "path", + "script_path", + "is_flow", + "enabled", + "postgres_resource_path" + ] + }, + "EditPostgresTrigger": { + "type": "object", + "properties": { + "replication_slot_name": { + "type": "string" + }, + "publication_name": { + "type": "string" + }, + "path": { + "type": "string" + }, + "script_path": { + "type": "string" + }, + "is_flow": { + "type": "boolean" + }, + "enabled": { + "type": "boolean" + }, + "postgres_resource_path": { + "type": "string" + }, + "publication": { + "$ref": "#/components/schemas/PublicationData" + } + }, + "required": [ + "path", + "script_path", + "is_flow", + "enabled", + "postgres_resource_path", + "publication_name", + "replication_slot_name" + ] + }, + "KafkaTrigger": { + "allOf": [ + { + "$ref": "#/components/schemas/TriggerExtraProperty" + } + ], + "type": "object", + "properties": { + "kafka_resource_path": { + "type": "string" + }, + "group_id": { + "type": "string" + }, + "topics": { + "type": "array", + "items": { + "type": "string" + } + }, + "server_id": { + "type": "string" + }, + "last_server_ping": { + "type": "string", + "format": "date-time" + }, + "error": { + "type": "string" + }, + "enabled": { + "type": "boolean" + } + }, + "required": [ + "kafka_resource_path", + "group_id", + "topics", + "enabled" + ] + }, + "NewKafkaTrigger": { + "type": "object", + "properties": { + "path": { + "type": "string" + }, + "script_path": { + "type": "string" + }, + "is_flow": { + "type": "boolean" + }, + "kafka_resource_path": { + "type": "string" + }, + "group_id": { + "type": "string" + }, + "topics": { + "type": "array", + "items": { + "type": "string" + } + }, + "enabled": { + "type": "boolean" + } + }, + "required": [ + "path", + "script_path", + "is_flow", + "kafka_resource_path", + "group_id", + "topics" + ] + }, + "EditKafkaTrigger": { + "type": "object", + "properties": { + "kafka_resource_path": { + "type": "string" + }, + "group_id": { + "type": "string" + }, + "topics": { + "type": "array", + "items": { + "type": "string" + } + }, + "path": { + "type": "string" + }, + "script_path": { + "type": "string" + }, + "is_flow": { + "type": "boolean" + } + }, + "required": [ + "path", + "script_path", + "kafka_resource_path", + "group_id", + "topics", + "is_flow" + ] + }, + "NatsTrigger": { + "allOf": [ + { + "$ref": "#/components/schemas/TriggerExtraProperty" + } + ], + "type": "object", + "properties": { + "nats_resource_path": { + "type": "string" + }, + "use_jetstream": { + "type": "boolean" + }, + "stream_name": { + "type": "string" + }, + "consumer_name": { + "type": "string" + }, + "subjects": { + "type": "array", + "items": { + "type": "string" + } + }, + "server_id": { + "type": "string" + }, + "last_server_ping": { + "type": "string", + "format": "date-time" + }, + "error": { + "type": "string" + }, + "enabled": { + "type": "boolean" + } + }, + "required": [ + "nats_resource_path", + "use_jetstream", + "subjects", + "enabled" + ] + }, + "NewNatsTrigger": { + "type": "object", + "properties": { + "path": { + "type": "string" + }, + "script_path": { + "type": "string" + }, + "is_flow": { + "type": "boolean" + }, + "nats_resource_path": { + "type": "string" + }, + "use_jetstream": { + "type": "boolean" + }, + "stream_name": { + "type": "string" + }, + "consumer_name": { + "type": "string" + }, + "subjects": { + "type": "array", + "items": { + "type": "string" + } + }, + "enabled": { + "type": "boolean" + } + }, + "required": [ + "path", + "script_path", + "is_flow", + "nats_resource_path", + "use_jetstream", + "subjects" + ] + }, + "EditNatsTrigger": { + "type": "object", + "properties": { + "nats_resource_path": { + "type": "string" + }, + "use_jetstream": { + "type": "boolean" + }, + "stream_name": { + "type": "string" + }, + "consumer_name": { + "type": "string" + }, + "subjects": { + "type": "array", + "items": { + "type": "string" + } + }, + "path": { + "type": "string" + }, + "script_path": { + "type": "string" + }, + "is_flow": { + "type": "boolean" + } + }, + "required": [ + "path", + "script_path", + "nats_resource_path", + "use_jetstream", + "subjects", + "is_flow" + ] + }, "Group": { "type": "object", "properties": { @@ -18850,12 +24873,19 @@ }, "username": { "type": "string" + }, + "color": { + "type": "string" + }, + "operator_settings": { + "$ref": "#/components/schemas/OperatorSettings" } }, "required": [ "id", "name", - "username" + "username", + "color" ] } } @@ -18876,6 +24906,9 @@ }, "username": { "type": "string" + }, + "color": { + "type": "string" } }, "required": [ @@ -18897,6 +24930,9 @@ }, "domain": { "type": "string" + }, + "color": { + "type": "string" } }, "required": [ @@ -18944,6 +24980,9 @@ "super_admin": { "type": "boolean" }, + "devops": { + "type": "boolean" + }, "verified": { "type": "boolean" }, @@ -18974,6 +25013,14 @@ }, { "$ref": "#/components/schemas/FlowMetadata" + }, + { + "type": "object", + "properties": { + "lock_error_logs": { + "type": "string" + } + } } ] }, @@ -19028,6 +25075,9 @@ }, "visible_to_runner_only": { "type": "boolean" + }, + "on_behalf_of_email": { + "type": "string" } }, "required": [ @@ -19066,6 +25116,9 @@ }, "visible_to_runner_only": { "type": "boolean" + }, + "on_behalf_of_email": { + "type": "string" } }, "required": [ @@ -19135,6 +25188,20 @@ "type": "object" } }, + "allowed_s3_keys": { + "type": "array", + "items": { + "type": "object", + "properties": { + "s3_path": { + "type": "string" + }, + "resource": { + "type": "string" + } + } + } + }, "execution_mode": { "type": "string", "enum": [ @@ -19189,6 +25256,9 @@ "publisher", "anonymous" ] + }, + "raw_app": { + "type": "boolean" } }, "required": [ @@ -19287,6 +25357,9 @@ "additionalProperties": { "type": "boolean" } + }, + "custom_path": { + "type": "string" } }, "required": [ @@ -19404,16 +25477,13 @@ ] }, "HubScriptKind": { - "name": "kind", - "schema": { - "type": "string", - "enum": [ - "script", - "failure", - "trigger", - "approval" - ] - } + "type": "string", + "enum": [ + "script", + "failure", + "trigger", + "approval" + ] }, "PolarsClientKwargs": { "type": "object", @@ -19588,7 +25658,8 @@ "resourcetype", "schedule", "user", - "group" + "group", + "trigger" ] } }, @@ -19619,7 +25690,8 @@ "app", "resource", "variable", - "secret" + "secret", + "trigger" ] } } @@ -19677,7 +25749,8 @@ "resourcetype", "schedule", "user", - "group" + "group", + "trigger" ] } } @@ -19775,25 +25848,7 @@ "type": "string" }, "language": { - "type": "string", - "enum": [ - "python3", - "deno", - "go", - "bash", - "powershell", - "postgresql", - "mysql", - "bigquery", - "snowflake", - "mssql", - "graphql", - "nativets", - "bun", - "php", - "rust", - "ansible" - ] + "$ref": "#/components/schemas/ScriptLang" } }, "required": [ @@ -20000,6 +26055,286 @@ "type": "boolean", "nullable": true, "description": "Acknowledgment status of the alert, can be true, false, or null if not set" + }, + "workspace_id": { + "type": "string", + "nullable": true, + "description": "Workspace id if the alert is in the scope of a workspace" + } + } + }, + "CaptureTriggerKind": { + "type": "string", + "enum": [ + "webhook", + "http", + "websocket", + "kafka", + "email", + "nats", + "postgres", + "sqs", + "mqtt", + "gcp" + ] + }, + "Capture": { + "type": "object", + "properties": { + "trigger_kind": { + "$ref": "#/components/schemas/CaptureTriggerKind" + }, + "main_args": {}, + "preprocessor_args": {}, + "id": { + "type": "integer" + }, + "created_at": { + "type": "string", + "format": "date-time" + } + }, + "required": [ + "trigger_kind", + "main_args", + "preprocessor_args", + "id", + "created_at" + ] + }, + "CaptureConfig": { + "type": "object", + "properties": { + "trigger_config": {}, + "trigger_kind": { + "$ref": "#/components/schemas/CaptureTriggerKind" + }, + "error": { + "type": "string" + }, + "last_server_ping": { + "type": "string", + "format": "date-time" + } + }, + "required": [ + "trigger_kind" + ] + }, + "OperatorSettings": { + "nullable": true, + "type": "object", + "required": [ + "runs", + "schedules", + "resources", + "variables", + "triggers", + "audit_logs", + "groups", + "folders", + "workers" + ], + "properties": { + "runs": { + "type": "boolean", + "description": "Whether operators can view runs" + }, + "schedules": { + "type": "boolean", + "description": "Whether operators can view schedules" + }, + "resources": { + "type": "boolean", + "description": "Whether operators can view resources" + }, + "variables": { + "type": "boolean", + "description": "Whether operators can view variables" + }, + "audit_logs": { + "type": "boolean", + "description": "Whether operators can view audit logs" + }, + "triggers": { + "type": "boolean", + "description": "Whether operators can view triggers" + }, + "groups": { + "type": "boolean", + "description": "Whether operators can view groups page" + }, + "folders": { + "type": "boolean", + "description": "Whether operators can view folders page" + }, + "workers": { + "type": "boolean", + "description": "Whether operators can view workers page" + } + } + }, + "TeamInfo": { + "type": "object", + "required": [ + "team_id", + "team_name", + "channels" + ], + "properties": { + "team_id": { + "type": "string", + "description": "The unique identifier of the Microsoft Teams team", + "example": "19:abc123def456@thread.tacv2" + }, + "team_name": { + "type": "string", + "description": "The display name of the Microsoft Teams team", + "example": "Engineering Team" + }, + "channels": { + "type": "array", + "description": "List of channels within the team", + "items": { + "$ref": "#/components/schemas/ChannelInfo" + } + } + } + }, + "ChannelInfo": { + "type": "object", + "required": [ + "channel_id", + "channel_name", + "tenant_id", + "service_url" + ], + "properties": { + "channel_id": { + "type": "string", + "description": "The unique identifier of the channel", + "example": "19:channel123@thread.tacv2" + }, + "channel_name": { + "type": "string", + "description": "The display name of the channel", + "example": "General" + }, + "tenant_id": { + "type": "string", + "description": "The Microsoft Teams tenant identifier", + "example": "12345678-1234-1234-1234-123456789012" + }, + "service_url": { + "type": "string", + "description": "The service URL for the channel", + "example": "https://smba.trafficmanager.net/amer/12345678-1234-1234-1234-123456789012/" + } + } + }, + "GithubInstallations": { + "type": "array", + "items": { + "type": "object", + "properties": { + "workspace_id": { + "type": "string" + }, + "installation_id": { + "type": "number" + }, + "account_id": { + "type": "string" + }, + "repositories": { + "type": "array", + "items": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "url": { + "type": "string" + } + }, + "required": [ + "name", + "url" + ] + } + } + }, + "required": [ + "installation_id", + "account_id", + "repositories" + ] + } + }, + "WorkspaceGithubInstallation": { + "type": "object", + "properties": { + "account_id": { + "type": "string" + }, + "installation_id": { + "type": "number" + } + }, + "required": [ + "account_id", + "installation_id" + ] + }, + "S3Object": { + "type": "object", + "properties": { + "s3": { + "type": "string" + }, + "filename": { + "type": "string" + }, + "storage": { + "type": "string" + }, + "presigned": { + "type": "string" + } + }, + "required": [ + "s3" + ] + }, + "TeamsChannel": { + "type": "object", + "required": [ + "team_id", + "team_name", + "channel_id", + "channel_name" + ], + "properties": { + "team_id": { + "type": "string", + "description": "Microsoft Teams team ID", + "minLength": 1 + }, + "team_name": { + "type": "string", + "description": "Microsoft Teams team name", + "minLength": 1 + }, + "channel_id": { + "type": "string", + "description": "Microsoft Teams channel ID", + "minLength": 1 + }, + "channel_name": { + "type": "string", + "description": "Microsoft Teams channel name", + "minLength": 1 } } }, @@ -20037,7 +26372,7 @@ "type" ] }, - "InputTransform": { + "schemas-InputTransform": { "oneOf": [ { "$ref": "#/components/schemas/StaticTransform" @@ -20060,7 +26395,7 @@ "input_transforms": { "type": "object", "additionalProperties": { - "$ref": "#/components/schemas/InputTransform" + "$ref": "#/components/schemas/schemas-InputTransform" } }, "content": { @@ -20080,6 +26415,7 @@ "bigquery", "snowflake", "mssql", + "oracledb", "graphql", "nativets", "php" @@ -20126,7 +26462,7 @@ "input_transforms": { "type": "object", "additionalProperties": { - "$ref": "#/components/schemas/InputTransform" + "$ref": "#/components/schemas/schemas-InputTransform" } }, "path": { @@ -20160,7 +26496,7 @@ "input_transforms": { "type": "object", "additionalProperties": { - "$ref": "#/components/schemas/InputTransform" + "$ref": "#/components/schemas/schemas-InputTransform" } }, "path": { @@ -20189,32 +26525,10 @@ "$ref": "#/components/schemas/FlowModuleValue" }, "stop_after_if": { - "type": "object", - "properties": { - "skip_if_stopped": { - "type": "boolean" - }, - "expr": { - "type": "string" - } - }, - "required": [ - "expr" - ] + "$ref": "#/components/schemas/StopAfterIf" }, "stop_after_all_iters_if": { - "type": "object", - "properties": { - "skip_if_stopped": { - "type": "boolean" - }, - "expr": { - "type": "string" - } - }, - "required": [ - "expr" - ] + "$ref": "#/components/schemas/StopAfterIf" }, "skip_if": { "type": "object", @@ -20228,7 +26542,7 @@ ] }, "sleep": { - "$ref": "#/components/schemas/InputTransform" + "$ref": "#/components/schemas/schemas-InputTransform" }, "cache_ttl": { "type": "number" @@ -20272,7 +26586,7 @@ "type": "boolean" }, "user_groups_required": { - "$ref": "#/components/schemas/InputTransform" + "$ref": "#/components/schemas/schemas-InputTransform" }, "self_approval_disabled": { "type": "boolean" @@ -20310,7 +26624,7 @@ } }, "iterator": { - "$ref": "#/components/schemas/InputTransform" + "$ref": "#/components/schemas/schemas-InputTransform" }, "skip_failures": { "type": "boolean" @@ -20515,6 +26829,23 @@ } } }, + "StopAfterIf": { + "type": "object", + "properties": { + "skip_if_stopped": { + "type": "boolean" + }, + "expr": { + "type": "string" + }, + "error_message": { + "type": "string" + } + }, + "required": [ + "expr" + ] + }, "Retry": { "type": "object", "properties": { diff --git a/backend/windmill-api/openapi-deref.yaml b/backend/windmill-api/openapi-deref.yaml index c8893f824c..4c13a9c7f8 100644 --- a/backend/windmill-api/openapi-deref.yaml +++ b/backend/windmill-api/openapi-deref.yaml @@ -1,6 +1,6 @@ openapi: 3.0.3 info: - version: 1.423.2 + version: 1.492.1 title: Windmill API contact: name: Windmill Team @@ -87,7 +87,7 @@ paths: - name: id in: path required: true - schema: &ref_30 + schema: &ref_34 type: integer responses: '200': @@ -233,24 +233,24 @@ paths: - name: before description: filter on started before (inclusive) timestamp in: query - schema: &ref_120 + schema: &ref_183 type: string format: date-time - name: after description: filter on created after (exclusive) timestamp in: query - schema: &ref_121 + schema: &ref_184 type: string format: date-time - name: username description: filter on exact username of user in: query - schema: &ref_130 + schema: &ref_189 type: string - name: operation description: filter on exact or prefix name of operation in: query - schema: &ref_131 + schema: &ref_190 type: string - name: operations in: query @@ -265,18 +265,23 @@ paths: - name: resource description: filter on exact or prefix name of resource in: query - schema: &ref_132 + schema: &ref_191 type: string - name: action_kind description: filter on type of operation in: query - schema: &ref_133 + schema: &ref_192 type: string enum: - Create - Update - Delete - Execute + - name: all_workspaces + in: query + description: get audit logs for all workspaces + schema: + type: boolean responses: '200': description: a list of audit logs @@ -302,12 +307,12 @@ paths: application/json: schema: type: object - properties: &ref_145 + properties: &ref_210 email: type: string password: type: string - required: &ref_146 + required: &ref_211 - email - password responses: @@ -375,6 +380,8 @@ paths: type: string is_admin: type: boolean + name: + type: string is_super_admin: type: boolean created_at: @@ -430,7 +437,7 @@ paths: application/json: schema: type: object - properties: &ref_147 + properties: &ref_212 is_admin: type: boolean operator: @@ -458,7 +465,7 @@ paths: - name: path in: path required: true - schema: &ref_23 + schema: &ref_26 type: string responses: '200': @@ -611,6 +618,8 @@ paths: properties: is_super_admin: type: boolean + is_devops: + type: boolean name: type: string responses: @@ -792,6 +801,44 @@ paths: text/plain: schema: type: string + /github_app/connected_repositories: + get: + summary: get connected repositories + operationId: getGlobalConnectedRepositories + tags: + - git_sync + responses: + '200': + description: connected repositories + content: + application/json: + schema: + type: array + items: &ref_331 + type: object + properties: + workspace_id: + type: string + installation_id: + type: number + account_id: + type: string + repositories: + type: array + items: + type: object + properties: + name: + type: string + url: + type: string + required: + - name + - url + required: + - installation_id + - account_id + - repositories /workspaces/list: get: summary: list all workspaces visible to me @@ -816,6 +863,8 @@ paths: type: string domain: type: string + color: + type: string required: &ref_8 - id - name @@ -846,7 +895,7 @@ paths: application/json: schema: type: object - properties: &ref_185 + properties: &ref_280 email: type: string workspaces: @@ -860,11 +909,55 @@ paths: type: string username: type: string + color: + type: string + operator_settings: + nullable: true + type: object + required: &ref_12 + - runs + - schedules + - resources + - variables + - triggers + - audit_logs + - groups + - folders + - workers + properties: &ref_13 + runs: + type: boolean + description: Whether operators can view runs + schedules: + type: boolean + description: Whether operators can view schedules + resources: + type: boolean + description: Whether operators can view resources + variables: + type: boolean + description: Whether operators can view variables + audit_logs: + type: boolean + description: Whether operators can view audit logs + triggers: + type: boolean + description: Whether operators can view triggers + groups: + type: boolean + description: Whether operators can view groups page + folders: + type: boolean + description: Whether operators can view folders page + workers: + type: boolean + description: Whether operators can view workers page required: - id - name - username - required: &ref_186 + - color + required: &ref_281 - email - workspaces /workspaces/list_as_superadmin: @@ -906,14 +999,16 @@ paths: application/json: schema: type: object - properties: &ref_187 + properties: &ref_282 id: type: string name: type: string username: type: string - required: &ref_188 + color: + type: string + required: &ref_283 - id - name responses: @@ -1143,29 +1238,46 @@ paths: content: application/json: schema: - type: array - items: - type: object - properties: &ref_230 - id: - type: integer - description: Unique identifier for the alert - alert_type: - type: string - description: Type of alert (e.g., critical_error) - message: - type: string - description: The message content of the alert - created_at: - type: string - format: date-time - description: Time when the alert was created - acknowledged: - type: boolean - nullable: true - description: >- - Acknowledgment status of the alert, can be true, false, - or null if not set + type: object + properties: + alerts: + type: array + items: + type: object + properties: &ref_29 + id: + type: integer + description: Unique identifier for the alert + alert_type: + type: string + description: Type of alert (e.g., critical_error) + message: + type: string + description: The message content of the alert + created_at: + type: string + format: date-time + description: Time when the alert was created + acknowledged: + type: boolean + nullable: true + description: >- + Acknowledgment status of the alert, can be true, + false, or null if not set + workspace_id: + type: string + nullable: true + description: >- + Workspace id if the alert is in the scope of a + workspace + total_rows: + type: integer + description: Total number of rows matching the query. + example: 100 + total_pages: + type: integer + description: Total number of pages based on the page size. + example: 10 /settings/critical_alerts/{id}/acknowledge: post: summary: Acknowledge a critical alert @@ -1356,12 +1468,12 @@ paths: type: array items: type: object - properties: &ref_223 + properties: &ref_318 name: type: string value: type: object - required: &ref_224 + required: &ref_319 - name - value /users/email: @@ -1383,9 +1495,15 @@ paths: operationId: refreshUserToken tags: - user + parameters: + - name: if_expiring_in_less_than_s + in: query + required: false + schema: + type: integer responses: '200': - description: free usage + description: new token content: text/plain: schema: @@ -1502,7 +1620,7 @@ paths: application/json: schema: type: object - properties: &ref_12 + properties: &ref_14 email: type: string login_type: @@ -1512,6 +1630,8 @@ paths: - github super_admin: type: boolean + devops: + type: boolean verified: type: boolean name: @@ -1522,7 +1642,7 @@ paths: type: string operator_only: type: boolean - required: &ref_13 + required: &ref_15 - email - login_type - super_admin @@ -1542,7 +1662,7 @@ paths: type: array items: type: object - properties: &ref_14 + properties: &ref_16 workspace_id: type: string email: @@ -1551,7 +1671,7 @@ paths: type: boolean operator: type: boolean - required: &ref_15 + required: &ref_17 - workspace_id - email - is_admin @@ -1576,6 +1696,154 @@ paths: type: object properties: *ref_10 required: *ref_11 + /w/{workspace}/github_app/token: + post: + summary: get github app token + operationId: getGithubAppToken + tags: + - workspace + parameters: + - name: workspace + in: path + required: true + schema: *ref_0 + requestBody: + description: jwt job token + required: true + content: + application/json: + schema: + type: object + properties: + job_token: + type: string + required: + - job_token + responses: + '200': + description: github app token + content: + application/json: + schema: + type: object + properties: + token: + type: string + required: + - token + /w/{workspace}/github_app/install_from_workspace: + post: + tags: + - Git Sync + summary: Install a GitHub installation from another workspace + operationId: installFromWorkspace + parameters: + - name: workspace + in: path + required: true + schema: *ref_0 + requestBody: + required: true + content: + application/json: + schema: + type: object + properties: + source_workspace_id: + type: string + description: The ID of the workspace containing the installation to copy + installation_id: + type: number + description: The ID of the GitHub installation to copy + required: + - source_workspace_id + - installation_id + responses: + '200': + description: Installation successfully copied + /w/{workspace}/github_app/installation/{installation_id}: + delete: + summary: Delete a GitHub installation from a workspace + operationId: deleteFromWorkspace + description: >- + Removes a GitHub installation from the specified workspace. Requires + admin privileges. + tags: + - Git Sync + parameters: + - name: workspace + in: path + required: true + schema: *ref_0 + - name: installation_id + in: path + required: true + schema: + type: integer + format: int64 + description: The ID of the GitHub installation to delete + responses: + '200': + description: Installation successfully deleted + /w/{workspace}/github_app/export/{installationId}: + get: + summary: Export GitHub installation JWT token + description: >- + Exports the JWT token for a specific GitHub installation in the + workspace + operationId: exportInstallation + tags: + - Git Sync + parameters: + - name: workspace + in: path + required: true + schema: + type: string + - name: installationId + in: path + required: true + schema: + type: integer + responses: + '200': + description: Successfully exported the JWT token + content: + application/json: + schema: + type: object + properties: + jwt_token: + type: string + /w/{workspace}/github_app/import: + post: + summary: Import GitHub installation from JWT token + description: >- + Imports a GitHub installation from a JWT token exported from another + instance + operationId: importInstallation + tags: + - Git Sync + parameters: + - name: workspace + in: path + required: true + schema: + type: string + requestBody: + required: true + content: + application/json: + schema: + type: object + required: + - jwt_token + properties: + jwt_token: + type: string + responses: + '200': + description: Successfully imported the installation /users/accept_invite: post: summary: accept invite to workspace @@ -1886,6 +2154,32 @@ paths: text/plain: schema: type: string + /w/{workspace}/workspaces/change_workspace_color: + post: + summary: change workspace id + operationId: changeWorkspaceColor + tags: + - workspace + parameters: + - name: workspace + in: path + required: true + schema: *ref_0 + requestBody: + content: + application/json: + schema: + type: object + properties: + color: + type: string + responses: + '200': + description: status + content: + text/plain: + schema: + type: string /w/{workspace}/users/whois/{username}: get: summary: whois @@ -1911,6 +2205,36 @@ paths: type: object properties: *ref_10 required: *ref_11 + /w/{workspace}/workspaces/operator_settings: + post: + operationId: updateOperatorSettings + summary: Update operator settings for a workspace + description: >- + Updates the operator settings for a specific workspace. Requires + workspace admin privileges. + tags: + - workspace + parameters: + - name: workspace + in: path + required: true + schema: *ref_0 + requestBody: + required: true + content: + application/json: + schema: + nullable: true + type: object + required: *ref_12 + properties: *ref_13 + responses: + '200': + description: Operator settings updated successfully + content: + text/plain: + schema: + type: string /users/exists/{email}: get: summary: exists email @@ -1959,8 +2283,8 @@ paths: type: array items: type: object - properties: *ref_12 - required: *ref_13 + properties: *ref_14 + required: *ref_15 /w/{workspace}/workspaces/list_pending_invites: get: summary: list pending invites for a workspace @@ -1981,8 +2305,8 @@ paths: type: array items: type: object - properties: *ref_14 - required: *ref_15 + properties: *ref_16 + required: *ref_17 /w/{workspace}/workspaces/get_settings: get: summary: get settings @@ -2010,6 +2334,12 @@ paths: type: string slack_command_script: type: string + teams_team_id: + type: string + teams_command_script: + type: string + teams_team_name: + type: string auto_invite_domain: type: string auto_invite_operator: @@ -2018,36 +2348,64 @@ paths: type: boolean plan: type: string - automatic_billing: - type: boolean customer_id: type: string webhook: type: string deploy_to: type: string - ai_resource: + ai_config: type: object - properties: &ref_16 - path: - type: string - provider: - type: string - required: &ref_17 - - path - - provider - code_completion_enabled: - type: boolean + properties: &ref_20 + providers: + type: object + additionalProperties: + type: object + properties: &ref_197 + resource_path: + type: string + models: + type: array + items: + type: string + required: &ref_198 + - resource_path + - models + default_model: + type: object + properties: &ref_18 + model: + type: string + provider: + type: string + enum: &ref_196 + - openai + - azure_openai + - anthropic + - mistral + - deepseek + - googleai + - groq + - openrouter + - togetherai + - customai + required: &ref_19 + - model + - provider + code_completion_model: + type: object + properties: *ref_18 + required: *ref_19 error_handler: type: string error_handler_extra_args: type: object - additionalProperties: &ref_18 {} + additionalProperties: &ref_21 {} error_handler_muted_on_cancel: type: boolean large_file_storage: type: object - properties: &ref_19 + properties: &ref_22 type: type: string enum: @@ -2081,7 +2439,7 @@ paths: type: boolean git_sync: type: object - properties: &ref_20 + properties: &ref_23 include_path: type: array items: @@ -2102,11 +2460,12 @@ paths: - schedule - user - group + - trigger repositories: type: array items: type: object - properties: &ref_207 + properties: &ref_302 script_path: type: string git_repo_resource_path: @@ -2131,12 +2490,13 @@ paths: - schedule - user - group - required: &ref_208 + - trigger + required: &ref_303 - script_path - git_repo_resource_path deploy_ui: type: object - properties: &ref_21 + properties: &ref_24 include_path: type: array items: @@ -2152,11 +2512,12 @@ paths: - resource - variable - secret + - trigger default_app: type: string default_scripts: type: object - properties: &ref_22 + properties: &ref_25 order: type: array items: @@ -2168,9 +2529,16 @@ paths: default_script_content: additionalProperties: type: string + mute_critical_alerts: + type: boolean + color: + type: string + operator_settings: + nullable: true + type: object + required: *ref_12 + properties: *ref_13 required: - - code_completion_enabled - - automatic_billing - error_handler_muted_on_cancel /w/{workspace}/workspaces/get_deploy_to: get: @@ -2234,17 +2602,40 @@ paths: type: boolean usage: type: number - seats: - type: number - automatic_billing: - type: boolean + owner: + type: string + status: + type: string required: - premium - - automatic_billing - /w/{workspace}/workspaces/set_automatic_billing: + - owner + /w/{workspace}/workspaces/threshold_alert: + get: + summary: get threshold alert info + operationId: getThresholdAlert + tags: + - workspace + parameters: + - name: workspace + in: path + required: true + schema: *ref_0 + responses: + '200': + description: status + content: + application/json: + schema: + type: object + properties: + threshold_alert_amount: + type: number + last_alert_sent: + type: string + format: date-time post: - summary: set automatic billing - operationId: setAutomaticBilling + summary: set threshold alert info + operationId: setThresholdAlert tags: - workspace parameters: @@ -2253,19 +2644,15 @@ paths: required: true schema: *ref_0 requestBody: - description: automatic billing + description: threshold alert info required: true content: application/json: schema: type: object properties: - automatic_billing: - type: boolean - seats: + threshold_alert_amount: type: number - required: - - automatic_billing responses: '200': description: status @@ -2301,6 +2688,118 @@ paths: text/plain: schema: type: string + /w/{workspace}/workspaces/edit_teams_command: + post: + summary: edit teams command + operationId: editTeamsCommand + tags: + - workspace + parameters: + - name: workspace + in: path + required: true + schema: *ref_0 + requestBody: + description: WorkspaceInvite + required: true + content: + application/json: + schema: + type: object + properties: + slack_command_script: + type: string + responses: + '200': + description: status + content: + text/plain: + schema: + type: string + /w/{workspace}/workspaces/available_teams_ids: + get: + summary: list available teams ids + operationId: listAvailableTeamsIds + tags: + - workspace + parameters: + - name: workspace + in: path + required: true + schema: *ref_0 + responses: + '200': + description: status + content: + application/json: + schema: + type: array + items: + type: object + properties: + team_name: + type: string + team_id: + type: string + /w/{workspace}/workspaces/available_teams_channels: + get: + summary: list available teams channels + operationId: listAvailableTeamsChannels + tags: + - workspace + parameters: + - name: workspace + in: path + required: true + schema: *ref_0 + responses: + '200': + description: status + content: + application/json: + schema: + type: array + items: + type: object + properties: + channel_name: + type: string + channel_id: + type: string + service_url: + type: string + tenant_id: + type: string + /w/{workspace}/workspaces/connect_teams: + post: + summary: connect teams + operationId: connectTeams + tags: + - workspace + parameters: + - name: workspace + in: path + required: true + schema: *ref_0 + requestBody: + description: connect teams + required: true + content: + application/json: + schema: + type: object + properties: + team_id: + type: string + team_name: + type: string + responses: + '200': + description: status + content: + text/plain: + schema: + type: string /w/{workspace}/workspaces/run_slack_message_test_job: post: summary: run a job that sends a message to Slack @@ -2336,6 +2835,41 @@ paths: properties: job_uuid: type: string + /w/{workspace}/workspaces/run_teams_message_test_job: + post: + summary: run a job that sends a message to Teams + operationId: runTeamsMessageTestJob + tags: + - workspace + parameters: + - name: workspace + in: path + required: true + schema: *ref_0 + requestBody: + description: path to hub script to run and its corresponding args + required: true + content: + application/json: + schema: + type: object + properties: + hub_script_path: + type: string + channel: + type: string + test_msg: + type: string + responses: + '200': + description: status + content: + text/json: + schema: + type: object + properties: + job_uuid: + type: string /w/{workspace}/workspaces/edit_deploy_to: post: summary: edit deploy to @@ -2441,15 +2975,7 @@ paths: application/json: schema: type: object - required: - - code_completion_enabled - properties: - ai_resource: - type: object - properties: *ref_16 - required: *ref_17 - code_completion_enabled: - type: boolean + properties: *ref_20 responses: '200': description: status @@ -2472,20 +2998,10 @@ paths: '200': description: status content: - text/plain: + application/json: schema: type: object - properties: - ai_provider: - type: string - exists_ai_resource: - type: boolean - code_completion_enabled: - type: boolean - required: - - ai_provider - - exists_ai_resource - - code_completion_enabled + properties: *ref_20 /w/{workspace}/workspaces/edit_error_handler: post: summary: edit error handler @@ -2509,7 +3025,7 @@ paths: type: string error_handler_extra_args: type: object - additionalProperties: *ref_18 + additionalProperties: *ref_21 error_handler_muted_on_cancel: type: boolean responses: @@ -2540,7 +3056,7 @@ paths: properties: large_file_storage: type: object - properties: *ref_19 + properties: *ref_22 responses: '200': description: status @@ -2568,7 +3084,7 @@ paths: properties: git_sync_settings: type: object - properties: *ref_20 + properties: *ref_23 responses: '200': description: status @@ -2596,7 +3112,7 @@ paths: properties: deploy_ui_settings: type: object - properties: *ref_21 + properties: *ref_24 responses: '200': description: status @@ -2648,7 +3164,7 @@ paths: application/json: schema: type: object - properties: *ref_22 + properties: *ref_25 responses: '200': description: status @@ -2673,7 +3189,7 @@ paths: application/json: schema: type: object - properties: *ref_22 + properties: *ref_25 /w/{workspace}/workspaces/set_environment_variable: post: summary: set environment variable @@ -2799,7 +3315,7 @@ paths: application/json: schema: type: object - properties: *ref_19 + properties: *ref_22 /w/{workspace}/workspaces/usage: get: summary: get usage @@ -2841,9 +3357,27 @@ paths: type: boolean websocket_used: type: boolean + kafka_used: + type: boolean + nats_used: + type: boolean + postgres_used: + type: boolean + mqtt_used: + type: boolean + gcp_used: + type: boolean + sqs_used: + type: boolean required: - http_routes_used - websocket_used + - kafka_used + - nats_used + - postgres_used + - mqtt_used + - gcp_used + - sqs_used /w/{workspace}/users/list: get: summary: list users @@ -2886,7 +3420,7 @@ paths: type: array items: type: object - properties: &ref_144 + properties: &ref_209 email: type: string executions: @@ -2947,7 +3481,7 @@ paths: application/json: schema: type: object - properties: &ref_148 + properties: &ref_213 label: type: string expiration: @@ -2979,7 +3513,7 @@ paths: application/json: schema: type: object - properties: &ref_149 + properties: &ref_214 label: type: string expiration: @@ -2989,7 +3523,7 @@ paths: type: string workspace_id: type: string - required: &ref_150 + required: &ref_215 - impersonate_email responses: '201': @@ -3045,7 +3579,7 @@ paths: type: array items: type: object - properties: &ref_40 + properties: &ref_47 label: type: string expiration: @@ -3065,7 +3599,7 @@ paths: type: string email: type: string - required: &ref_41 + required: &ref_48 - token_prefix - created_at - last_used_at @@ -3114,7 +3648,7 @@ paths: application/json: schema: type: object - properties: &ref_153 + properties: &ref_218 path: type: string value: @@ -3130,7 +3664,7 @@ paths: expires_at: type: string format: date-time - required: &ref_154 + required: &ref_219 - path - value - is_secret @@ -3181,7 +3715,7 @@ paths: - name: path in: path required: true - schema: *ref_23 + schema: *ref_26 responses: '200': description: variable deleted @@ -3203,7 +3737,7 @@ paths: - name: path in: path required: true - schema: *ref_23 + schema: *ref_26 - name: already_encrypted in: query schema: @@ -3215,7 +3749,7 @@ paths: application/json: schema: type: object - properties: &ref_155 + properties: &ref_220 path: type: string value: @@ -3245,7 +3779,7 @@ paths: - name: path in: path required: true - schema: *ref_23 + schema: *ref_26 - name: decrypt_secret description: | ask to decrypt secret if this variable is secret @@ -3267,7 +3801,7 @@ paths: application/json: schema: type: object - properties: &ref_24 + properties: &ref_27 workspace_id: type: string path: @@ -3297,7 +3831,7 @@ paths: expires_at: type: string format: date-time - required: &ref_25 + required: &ref_28 - workspace_id - path - is_secret @@ -3316,7 +3850,7 @@ paths: - name: path in: path required: true - schema: *ref_23 + schema: *ref_26 responses: '200': description: variable @@ -3338,7 +3872,7 @@ paths: - name: path in: path required: true - schema: *ref_23 + schema: *ref_26 responses: '200': description: variable @@ -3378,8 +3912,8 @@ paths: type: array items: type: object - properties: *ref_24 - required: *ref_25 + properties: *ref_27 + required: *ref_28 /w/{workspace}/variables/list_contextual: get: summary: list contextual variables @@ -3400,7 +3934,7 @@ paths: type: array items: type: object - properties: &ref_151 + properties: &ref_216 name: type: string value: @@ -3409,11 +3943,141 @@ paths: type: string is_custom: type: boolean - required: &ref_152 + required: &ref_217 - name - value - description - is_custom + /w/{workspace}/workspaces/critical_alerts: + get: + summary: Get all critical alerts for this workspace + operationId: workspaceGetCriticalAlerts + tags: + - setting + parameters: + - name: workspace + in: path + required: true + schema: *ref_0 + - in: query + name: page + schema: + type: integer + default: 1 + description: The page number to retrieve (minimum value is 1) + - in: query + name: page_size + schema: + type: integer + default: 10 + maximum: 100 + description: Number of alerts per page (maximum is 100) + - in: query + name: acknowledged + schema: + type: boolean + nullable: true + description: >- + Filter by acknowledgment status; true for acknowledged, false for + unacknowledged, and omit for all alerts + responses: + '200': + description: Successfully retrieved all critical alerts + content: + application/json: + schema: + type: object + properties: + alerts: + type: array + items: + type: object + properties: *ref_29 + total_rows: + type: integer + description: Total number of rows matching the query. + example: 100 + total_pages: + type: integer + description: Total number of pages based on the page size. + example: 10 + /w/{workspace}/workspaces/critical_alerts/{id}/acknowledge: + post: + summary: Acknowledge a critical alert for this workspace + operationId: workspaceAcknowledgeCriticalAlert + tags: + - setting + parameters: + - name: workspace + in: path + required: true + schema: *ref_0 + - in: path + name: id + required: true + schema: + type: integer + description: The ID of the critical alert to acknowledge + responses: + '200': + description: Successfully acknowledged the critical alert + content: + application/json: + schema: + type: string + example: Critical alert acknowledged + /w/{workspace}/workspaces/critical_alerts/acknowledge_all: + post: + summary: Acknowledge all unacknowledged critical alerts for this workspace + operationId: workspaceAcknowledgeAllCriticalAlerts + tags: + - setting + parameters: + - name: workspace + in: path + required: true + schema: *ref_0 + responses: + '200': + description: Successfully acknowledged all unacknowledged critical alerts. + content: + application/json: + schema: + type: string + example: All unacknowledged critical alerts acknowledged + /w/{workspace}/workspaces/critical_alerts/mute: + post: + summary: Mute critical alert UI for this workspace + operationId: workspaceMuteCriticalAlertsUI + tags: + - setting + parameters: + - name: workspace + in: path + required: true + schema: *ref_0 + requestBody: + description: Boolean flag to mute critical alerts. + required: true + content: + application/json: + schema: + type: object + properties: + mute_critical_alerts: + type: boolean + description: Whether critical alerts should be muted. + example: true + responses: + '200': + description: Successfully updated mute critical alert settings. + content: + application/json: + schema: + type: string + example: >- + Updated mute critical alert UI settings for workspace: + workspace_id /oauth/login_callback/{client_name}: post: security: [] @@ -3425,7 +4089,7 @@ paths: - name: client_name in: path required: true - schema: &ref_26 + schema: &ref_30 type: string requestBody: description: Partially filled script @@ -3526,7 +4190,7 @@ paths: - name: client_name in: path required: true - schema: *ref_26 + schema: *ref_30 requestBody: description: code endpoint required: true @@ -3549,7 +4213,7 @@ paths: application/json: schema: type: object - properties: &ref_200 + properties: &ref_295 access_token: type: string expires_in: @@ -3560,7 +4224,7 @@ paths: type: array items: type: string - required: &ref_201 + required: &ref_296 - access_token /w/{workspace}/oauth/create_account: post: @@ -3611,7 +4275,7 @@ paths: - name: id in: path required: true - schema: &ref_27 + schema: &ref_31 type: integer requestBody: description: variable path @@ -3646,7 +4310,7 @@ paths: - name: id in: path required: true - schema: *ref_27 + schema: *ref_31 responses: '200': description: disconnected client @@ -3672,6 +4336,24 @@ paths: text/plain: schema: type: string + /w/{workspace}/oauth/disconnect_teams: + post: + summary: disconnect teams + operationId: disconnectTeams + tags: + - oauth + parameters: + - name: workspace + in: path + required: true + schema: *ref_0 + responses: + '200': + description: disconnected teams + content: + text/plain: + schema: + type: string /oauth/list_logins: get: summary: list oauth logins @@ -3743,6 +4425,95 @@ paths: type: array items: type: string + /teams/sync: + post: + operationId: syncTeams + summary: synchronize Microsoft Teams information (teams/channels) + tags: + - teams + responses: + '200': + description: Teams information successfully synchronized + content: + application/json: + schema: + type: array + items: + type: object + required: &ref_327 + - team_id + - team_name + - channels + properties: &ref_328 + team_id: + type: string + description: The unique identifier of the Microsoft Teams team + example: 19:abc123def456@thread.tacv2 + team_name: + type: string + description: The display name of the Microsoft Teams team + example: Engineering Team + channels: + type: array + description: List of channels within the team + items: + type: object + required: &ref_329 + - channel_id + - channel_name + - tenant_id + - service_url + properties: &ref_330 + channel_id: + type: string + description: The unique identifier of the channel + example: 19:channel123@thread.tacv2 + channel_name: + type: string + description: The display name of the channel + example: General + tenant_id: + type: string + description: The Microsoft Teams tenant identifier + example: 12345678-1234-1234-1234-123456789012 + service_url: + type: string + description: The service URL for the channel + example: >- + https://smba.trafficmanager.net/amer/12345678-1234-1234-1234-123456789012/ + /teams/activities: + post: + summary: send update to Microsoft Teams activity + description: Respond to a Microsoft Teams activity after a workspace command is run + operationId: sendMessageToConversation + tags: + - teams + requestBody: + required: true + content: + application/json: + schema: + type: object + required: + - conversation_id + - text + properties: + conversation_id: + type: string + description: The ID of the Teams conversation/activity + success: + type: boolean + description: Used for styling the card conditionally + default: true + text: + type: string + description: The message text to be sent in the Teams card + card_block: + type: object + description: The card block to be sent in the Teams card + responses: + '200': + description: Activity processed successfully /w/{workspace}/resources/create: post: summary: create resource @@ -3765,7 +4536,7 @@ paths: application/json: schema: type: object - properties: &ref_162 + properties: &ref_227 path: type: string value: {} @@ -3773,7 +4544,7 @@ paths: type: string resource_type: type: string - required: &ref_163 + required: &ref_228 - path - value - resource_type @@ -3798,7 +4569,7 @@ paths: - name: path in: path required: true - schema: *ref_23 + schema: *ref_26 responses: '200': description: resource deleted @@ -3820,7 +4591,7 @@ paths: - name: path in: path required: true - schema: *ref_23 + schema: *ref_26 requestBody: description: updated resource required: true @@ -3828,7 +4599,7 @@ paths: application/json: schema: type: object - properties: &ref_164 + properties: &ref_229 path: type: string description: @@ -3855,7 +4626,7 @@ paths: - name: path in: path required: true - schema: *ref_23 + schema: *ref_26 requestBody: description: updated resource required: true @@ -3886,7 +4657,7 @@ paths: - name: path in: path required: true - schema: *ref_23 + schema: *ref_26 responses: '200': description: resource @@ -3894,7 +4665,7 @@ paths: application/json: schema: type: object - properties: &ref_165 + properties: &ref_230 workspace_id: type: string path: @@ -3915,7 +4686,7 @@ paths: edited_at: type: string format: date-time - required: &ref_166 + required: &ref_231 - path - resource_type - is_oauth @@ -3933,7 +4704,7 @@ paths: - name: path in: path required: true - schema: *ref_23 + schema: *ref_26 - name: job_id description: job id in: query @@ -3960,7 +4731,7 @@ paths: - name: path in: path required: true - schema: *ref_23 + schema: *ref_26 responses: '200': description: resource value @@ -3981,7 +4752,7 @@ paths: - name: path in: path required: true - schema: *ref_23 + schema: *ref_26 responses: '200': description: does resource exists @@ -4031,7 +4802,7 @@ paths: type: array items: type: object - properties: &ref_167 + properties: &ref_232 workspace_id: type: string path: @@ -4062,7 +4833,7 @@ paths: edited_at: type: string format: date-time - required: &ref_168 + required: &ref_233 - path - resource_type - is_oauth @@ -4109,7 +4880,7 @@ paths: - name: name in: path required: true - schema: &ref_103 + schema: &ref_163 type: string responses: '200': @@ -4146,7 +4917,7 @@ paths: application/json: schema: type: object - properties: &ref_28 + properties: &ref_32 workspace_id: type: string name: @@ -4161,7 +4932,7 @@ paths: format: date-time format_extension: type: string - required: &ref_29 + required: &ref_33 - name responses: '201': @@ -4201,7 +4972,7 @@ paths: - name: path in: path required: true - schema: *ref_23 + schema: *ref_26 responses: '200': description: resource_type deleted @@ -4223,7 +4994,7 @@ paths: - name: path in: path required: true - schema: *ref_23 + schema: *ref_26 requestBody: description: updated resource_type required: true @@ -4231,7 +5002,7 @@ paths: application/json: schema: type: object - properties: &ref_169 + properties: &ref_234 schema: {} description: type: string @@ -4256,7 +5027,7 @@ paths: - name: path in: path required: true - schema: *ref_23 + schema: *ref_26 responses: '200': description: resource_type deleted @@ -4264,8 +5035,8 @@ paths: application/json: schema: type: object - properties: *ref_28 - required: *ref_29 + properties: *ref_32 + required: *ref_33 /w/{workspace}/resources/type/exists/{path}: get: summary: does resource_type exists @@ -4280,7 +5051,7 @@ paths: - name: path in: path required: true - schema: *ref_23 + schema: *ref_26 responses: '200': description: does resource_type exist @@ -4308,8 +5079,8 @@ paths: type: array items: type: object - properties: *ref_28 - required: *ref_29 + properties: *ref_32 + required: *ref_33 /w/{workspace}/resources/type/listnames: get: summary: list resource_types names @@ -4448,7 +5219,7 @@ paths: - name: id in: path required: true - schema: *ref_30 + schema: *ref_34 responses: '200': description: flow @@ -4459,51 +5230,51 @@ paths: properties: flow: type: object - properties: &ref_54 + properties: &ref_61 summary: type: string description: type: string value: type: object - properties: &ref_66 + properties: &ref_78 modules: type: array items: type: object - properties: &ref_33 + properties: &ref_37 id: type: string value: - oneOf: &ref_251 + oneOf: &ref_352 - type: object - properties: &ref_235 + properties: &ref_336 input_transforms: type: object additionalProperties: - oneOf: &ref_31 + oneOf: &ref_35 - type: object - properties: &ref_231 + properties: &ref_332 value: {} type: type: string enum: - javascript - required: &ref_232 + required: &ref_333 - expr - type - type: object - properties: &ref_233 + properties: &ref_334 expr: type: string type: type: string enum: - javascript - required: &ref_234 + required: &ref_335 - expr - type - discriminator: &ref_32 + discriminator: &ref_36 propertyName: type mapping: static: '#/components/schemas/StaticTransform' @@ -4524,6 +5295,7 @@ paths: - bigquery - snowflake - mssql + - oracledb - graphql - nativets - php @@ -4545,18 +5317,18 @@ paths: type: string is_trigger: type: boolean - required: &ref_236 + required: &ref_337 - type - content - language - input_transforms - type: object - properties: &ref_237 + properties: &ref_338 input_transforms: type: object additionalProperties: - oneOf: *ref_31 - discriminator: *ref_32 + oneOf: *ref_35 + discriminator: *ref_36 path: type: string hash: @@ -4569,40 +5341,40 @@ paths: type: string is_trigger: type: boolean - required: &ref_238 + required: &ref_339 - type - path - input_transforms - type: object - properties: &ref_239 + properties: &ref_340 input_transforms: type: object additionalProperties: - oneOf: *ref_31 - discriminator: *ref_32 + oneOf: *ref_35 + discriminator: *ref_36 path: type: string type: type: string enum: - flow - required: &ref_240 + required: &ref_341 - type - path - input_transforms - type: object - properties: &ref_241 + properties: &ref_342 modules: type: array items: type: object - properties: *ref_33 - required: &ref_34 + properties: *ref_37 + required: &ref_38 - value - id iterator: - oneOf: *ref_31 - discriminator: *ref_32 + oneOf: *ref_35 + discriminator: *ref_36 skip_failures: type: boolean type: @@ -4613,19 +5385,19 @@ paths: type: boolean parallelism: type: integer - required: &ref_242 + required: &ref_343 - modules - iterator - skip_failures - type - type: object - properties: &ref_243 + properties: &ref_344 modules: type: array items: type: object - properties: *ref_33 - required: *ref_34 + properties: *ref_37 + required: *ref_38 skip_failures: type: boolean type: @@ -4636,12 +5408,12 @@ paths: type: boolean parallelism: type: integer - required: &ref_244 + required: &ref_345 - modules - skip_failures - type - type: object - properties: &ref_245 + properties: &ref_346 branches: type: array items: @@ -4655,8 +5427,8 @@ paths: type: array items: type: object - properties: *ref_33 - required: *ref_34 + properties: *ref_37 + required: *ref_38 required: - modules - expr @@ -4664,20 +5436,20 @@ paths: type: array items: type: object - properties: *ref_33 - required: *ref_34 + properties: *ref_37 + required: *ref_38 required: - modules type: type: string enum: - branchone - required: &ref_246 + required: &ref_347 - branches - default - type - type: object - properties: &ref_247 + properties: &ref_348 branches: type: array items: @@ -4691,8 +5463,8 @@ paths: type: array items: type: object - properties: *ref_33 - required: *ref_34 + properties: *ref_37 + required: *ref_38 required: - modules - expr @@ -4702,20 +5474,20 @@ paths: - branchall parallel: type: boolean - required: &ref_248 + required: &ref_349 - branches - type - type: object - properties: &ref_249 + properties: &ref_350 type: type: string enum: - identity flow: type: boolean - required: &ref_250 + required: &ref_351 - type - discriminator: &ref_252 + discriminator: &ref_353 propertyName: type mapping: rawscript: '#/components/schemas/RawScript' @@ -4728,22 +5500,19 @@ paths: identity: '#/components/schemas/Identity' stop_after_if: type: object - properties: + properties: &ref_39 skip_if_stopped: type: boolean expr: type: string - required: + error_message: + type: string + required: &ref_40 - expr stop_after_all_iters_if: type: object - properties: - skip_if_stopped: - type: boolean - expr: - type: string - required: - - expr + properties: *ref_39 + required: *ref_40 skip_if: type: object properties: @@ -4752,8 +5521,8 @@ paths: required: - expr sleep: - oneOf: *ref_31 - discriminator: *ref_32 + oneOf: *ref_35 + discriminator: *ref_36 cache_ttl: type: number timeout: @@ -4783,8 +5552,8 @@ paths: user_auth_required: type: boolean user_groups_required: - oneOf: *ref_31 - discriminator: *ref_32 + oneOf: *ref_35 + discriminator: *ref_36 self_approval_disabled: type: boolean hide_cancel: @@ -4797,7 +5566,7 @@ paths: type: boolean retry: type: object - properties: &ref_95 + properties: &ref_114 constant: type: object properties: @@ -4818,15 +5587,15 @@ paths: type: integer minimum: 0 maximum: 100 - required: *ref_34 + required: *ref_38 failure_module: type: object - properties: *ref_33 - required: *ref_34 + properties: *ref_37 + required: *ref_38 preprocessor_module: type: object - properties: *ref_33 - required: *ref_34 + properties: *ref_37 + required: *ref_38 same_worker: type: boolean concurrent_limit: @@ -4843,11 +5612,11 @@ paths: type: number early_return: type: string - required: &ref_67 + required: &ref_79 - modules schema: type: object - required: &ref_55 + required: &ref_62 - summary - value /apps/hub/list: @@ -4900,7 +5669,7 @@ paths: - name: id in: path required: true - schema: *ref_30 + schema: *ref_34 responses: '200': description: app @@ -4920,6 +5689,108 @@ paths: - value required: - app + /apps_u/public_app_by_custom_path/{custom_path}: + get: + summary: get public app by custom path + operationId: getPublicAppByCustomPath + tags: + - app + parameters: + - name: custom_path + in: path + required: true + schema: &ref_74 + type: string + responses: + '200': + description: app details + content: + application/json: + schema: + allOf: + - type: object + properties: &ref_69 + id: + type: integer + workspace_id: + type: string + path: + type: string + summary: + type: string + versions: + type: array + items: + type: integer + created_by: + type: string + created_at: + type: string + format: date-time + value: + type: object + policy: + type: object + properties: &ref_68 + triggerables: + type: object + additionalProperties: + type: object + triggerables_v2: + type: object + additionalProperties: + type: object + s3_inputs: + type: array + items: + type: object + allowed_s3_keys: + type: array + items: + type: object + properties: + s3_path: + type: string + resource: + type: string + execution_mode: + type: string + enum: + - viewer + - publisher + - anonymous + on_behalf_of: + type: string + on_behalf_of_email: + type: string + execution_mode: + type: string + enum: + - viewer + - publisher + - anonymous + extra_perms: + type: object + additionalProperties: + type: boolean + custom_path: + type: string + required: &ref_70 + - id + - workspace_id + - path + - summary + - versions + - created_by + - created_at + - value + - policy + - execution_mode + - extra_perms + - type: object + properties: + workspace_id: + type: string /scripts/hub/get/{path}: get: summary: get hub script content by path @@ -4930,7 +5801,7 @@ paths: - name: path in: path required: true - schema: &ref_35 + schema: &ref_41 type: string responses: '200': @@ -4949,7 +5820,7 @@ paths: - name: path in: path required: true - schema: *ref_35 + schema: *ref_41 responses: '200': description: script details @@ -5019,14 +5890,12 @@ paths: version_id: type: number kind: - name: kind - schema: &ref_36 - type: string - enum: - - script - - failure - - trigger - - approval + type: string + enum: &ref_42 + - script + - failure + - trigger + - approval votes: type: number views: @@ -5092,8 +5961,8 @@ paths: app: type: string kind: - name: kind - schema: *ref_36 + type: string + enum: *ref_42 score: type: number required: @@ -5154,12 +6023,12 @@ paths: - name: order_desc description: order by desc order (default true) in: query - schema: &ref_52 + schema: &ref_59 type: boolean - name: created_by description: mask to filter exact matching user creator in: query - schema: &ref_53 + schema: &ref_60 type: string - name: path_start description: mask to filter matching starting path @@ -5178,8 +6047,7 @@ paths: type: string - name: last_parent_hash description: > - mask to filter scripts whom last parent in the chain has exact - hash. + mask to filter scripts whom last parent in the chain has exact hash. Beware that each script stores only a limited number of parents. Hence @@ -5214,7 +6082,7 @@ paths: when multiple archived hash share the same path, only the ones with the latest create_at - are + are ed. in: query @@ -5273,7 +6141,7 @@ paths: type: array items: type: object - properties: &ref_37 + properties: &ref_44 workspace_id: type: string hash: @@ -5316,7 +6184,7 @@ paths: type: string language: type: string - enum: + enum: &ref_43 - python3 - deno - go @@ -5327,12 +6195,16 @@ paths: - bigquery - snowflake - mssql + - oracledb - graphql - nativets - bun - php - rust - ansible + - csharp + - nu + - java kind: type: string enum: @@ -5341,6 +6213,7 @@ paths: - trigger - command - approval + - preprocessor starred: type: boolean tag: @@ -5381,7 +6254,9 @@ paths: type: string has_preprocessor: type: boolean - required: &ref_38 + on_behalf_of_email: + type: string + required: &ref_45 - hash - path - summary @@ -5479,7 +6354,7 @@ paths: - name: path in: path required: true - schema: *ref_35 + schema: *ref_41 responses: '200': description: draft deleted @@ -5505,7 +6380,7 @@ paths: application/json: schema: type: object - properties: &ref_42 + properties: &ref_49 path: type: string parent_hash: @@ -5524,23 +6399,7 @@ paths: type: string language: type: string - enum: - - python3 - - deno - - go - - bash - - powershell - - postgresql - - mysql - - bigquery - - snowflake - - mssql - - graphql - - nativets - - bun - - php - - rust - - ansible + enum: *ref_43 kind: type: string enum: @@ -5549,6 +6408,7 @@ paths: - trigger - command - approval + - preprocessor tag: type: string draft_only: @@ -5587,7 +6447,9 @@ paths: type: string has_preprocessor: type: boolean - required: &ref_43 + on_behalf_of_email: + type: string + required: &ref_50 - path - summary - description @@ -5614,7 +6476,7 @@ paths: - name: path in: path required: true - schema: *ref_35 + schema: *ref_41 requestBody: description: Workspace error handler enabled required: true @@ -5640,6 +6502,17 @@ paths: operationId: getCustomTags tags: - worker + parameters: + - name: workspace + in: query + schema: + type: string + required: false + - name: show_workspace_restriction + in: query + schema: + type: boolean + required: false responses: '200': description: list of custom tags @@ -5691,7 +6564,7 @@ paths: - name: path in: path required: true - schema: *ref_35 + schema: *ref_41 responses: '200': description: script archived @@ -5713,7 +6586,7 @@ paths: - name: hash in: path required: true - schema: &ref_39 + schema: &ref_46 type: string responses: '200': @@ -5722,8 +6595,8 @@ paths: application/json: schema: type: object - properties: *ref_37 - required: *ref_38 + properties: *ref_44 + required: *ref_45 /w/{workspace}/scripts/delete/h/{hash}: post: summary: delete script by hash (erase content but keep hash, require admin) @@ -5738,7 +6611,7 @@ paths: - name: hash in: path required: true - schema: *ref_39 + schema: *ref_46 responses: '200': description: script details @@ -5746,11 +6619,11 @@ paths: application/json: schema: type: object - properties: *ref_37 - required: *ref_38 + properties: *ref_44 + required: *ref_45 /w/{workspace}/scripts/delete/p/{path}: post: - summary: delete all scripts at a given path (require admin) + summary: delete script at a given path (require admin) operationId: deleteScriptByPath tags: - script @@ -5762,7 +6635,12 @@ paths: - name: path in: path required: true - schema: *ref_35 + schema: *ref_41 + - name: keep_captures + description: keep captures + in: query + schema: + type: boolean responses: '200': description: script path @@ -5784,7 +6662,7 @@ paths: - name: path in: path required: true - schema: *ref_35 + schema: *ref_41 - name: with_starred_info in: query schema: @@ -5796,8 +6674,8 @@ paths: application/json: schema: type: object - properties: *ref_37 - required: *ref_38 + properties: *ref_44 + required: *ref_45 /w/{workspace}/scripts/get_triggers_count/{path}: get: summary: get triggers count of script @@ -5812,7 +6690,7 @@ paths: - name: path in: path required: true - schema: *ref_35 + schema: *ref_41 responses: '200': description: triggers count @@ -5820,7 +6698,7 @@ paths: application/json: schema: type: object - properties: &ref_59 + properties: &ref_66 primary_schedule: type: object properties: @@ -5836,6 +6714,18 @@ paths: type: number websocket_count: type: number + postgres_count: + type: number + kafka_count: + type: number + nats_count: + type: number + mqtt_count: + type: number + gcp_count: + type: number + sqs_count: + type: number /w/{workspace}/scripts/list_tokens/{path}: get: summary: get tokens with script scope @@ -5850,7 +6740,7 @@ paths: - name: path in: path required: true - schema: *ref_35 + schema: *ref_41 responses: '200': description: tokens list @@ -5860,8 +6750,8 @@ paths: type: array items: type: object - properties: *ref_40 - required: *ref_41 + properties: *ref_47 + required: *ref_48 /w/{workspace}/scripts/get/draft/{path}: get: summary: get script by path with draft @@ -5876,23 +6766,23 @@ paths: - name: path in: path required: true - schema: *ref_35 + schema: *ref_41 responses: '200': description: script details content: application/json: schema: - allOf: &ref_137 + allOf: &ref_202 - type: object - properties: *ref_42 - required: *ref_43 + properties: *ref_49 + required: *ref_50 - type: object properties: draft: type: object - properties: *ref_42 - required: *ref_43 + properties: *ref_49 + required: *ref_50 hash: type: string required: @@ -5911,7 +6801,7 @@ paths: - name: path in: path required: true - schema: *ref_35 + schema: *ref_41 responses: '200': description: script history @@ -5921,13 +6811,37 @@ paths: type: array items: type: object - properties: &ref_44 + properties: &ref_51 script_hash: type: string deployment_msg: type: string - required: &ref_45 + required: &ref_52 - script_hash + /w/{workspace}/scripts/list_paths_from_workspace_runnable/{path}: + get: + summary: list script paths using provided script as a relative import + operationId: listScriptPathsFromWorkspaceRunnable + tags: + - script + parameters: + - name: workspace + in: path + required: true + schema: *ref_0 + - name: path + in: path + required: true + schema: *ref_41 + responses: + '200': + description: list of script paths + content: + application/json: + schema: + type: array + items: + type: string /w/{workspace}/scripts/get_latest_version/{path}: get: summary: get scripts's latest version (hash) @@ -5940,7 +6854,7 @@ paths: - name: path in: path required: true - schema: *ref_35 + schema: *ref_41 tags: - script responses: @@ -5948,11 +6862,10 @@ paths: description: Script version/hash content: application/json: - required: false schema: type: object - properties: *ref_44 - required: *ref_45 + properties: *ref_51 + required: *ref_52 /w/{workspace}/scripts/history_update/h/{hash}/p/{path}: post: summary: update history of a script @@ -5967,11 +6880,11 @@ paths: - name: hash in: path required: true - schema: *ref_39 + schema: *ref_46 - name: path in: path required: true - schema: *ref_35 + schema: *ref_41 requestBody: description: Script deployment message required: true @@ -6003,7 +6916,7 @@ paths: - name: path in: path required: true - schema: *ref_35 + schema: *ref_41 responses: '200': description: script content @@ -6027,12 +6940,12 @@ paths: - name: token in: path required: true - schema: &ref_126 + schema: &ref_187 type: string - name: path in: path required: true - schema: *ref_35 + schema: *ref_41 responses: '200': description: script content @@ -6054,7 +6967,7 @@ paths: - name: path in: path required: true - schema: *ref_35 + schema: *ref_41 responses: '200': description: does it exists @@ -6076,7 +6989,7 @@ paths: - name: hash in: path required: true - schema: *ref_39 + schema: *ref_46 - name: with_starred_info in: query schema: @@ -6088,8 +7001,8 @@ paths: application/json: schema: type: object - properties: *ref_37 - required: *ref_38 + properties: *ref_44 + required: *ref_45 /w/{workspace}/scripts/raw/h/{path}: get: summary: raw script by hash @@ -6104,7 +7017,7 @@ paths: - name: path in: path required: true - schema: *ref_35 + schema: *ref_41 responses: '200': description: script content @@ -6126,7 +7039,7 @@ paths: - name: hash in: path required: true - schema: *ref_39 + schema: *ref_46 responses: '200': description: script details @@ -6139,6 +7052,68 @@ paths: type: string lock_error_logs: type: string + /w/{workspace}/jobs/list_selected_job_groups: + post: + summary: list selected jobs script/flow schemas grouped by (kind, path) + operationId: listSelectedJobGroups + tags: + - job + parameters: + - name: workspace + in: path + required: true + schema: *ref_0 + requestBody: + description: script args + required: true + content: + application/json: + schema: + type: array + items: + type: string + format: uuid + responses: + '200': + description: result + content: + text/plain: + schema: + type: array + items: + type: object + properties: + kind: + type: string + enum: + - script + - flow + script_path: + type: string + latest_schema: + type: object + schemas: + type: array + items: + type: object + properties: + schema: + type: object + script_hash: + type: string + job_ids: + type: array + items: + type: string + required: + - schema + - script_hash + - job_ids + required: + - kind + - script_path + - latest_schema + - schemas /w/{workspace}/jobs/run/p/{path}: post: summary: run script by path @@ -6153,7 +7128,7 @@ paths: - name: path in: path required: true - schema: *ref_35 + schema: *ref_41 - name: scheduled_for description: when to schedule this job (leave empty for immediate run) in: query @@ -6175,20 +7150,20 @@ paths: The parent job that is at the origin and responsible for the execution of this script if any in: query - schema: &ref_46 + schema: &ref_53 type: string format: uuid - name: tag description: Override the tag to use in: query - schema: &ref_48 + schema: &ref_55 type: string - name: cache_ttl description: >- Override the cache time to live (in seconds). Can not be used to disable caching, only override with a new cache ttl in: query - schema: &ref_49 + schema: &ref_56 type: string - name: job_id description: >- @@ -6197,7 +7172,7 @@ paths: queue or as a completed job, the request to create one will fail (Bad Request) in: query - schema: &ref_47 + schema: &ref_54 type: string format: uuid - name: invisible_to_owner @@ -6212,7 +7187,7 @@ paths: application/json: schema: type: object - additionalProperties: *ref_18 + additionalProperties: *ref_21 responses: '201': description: job created @@ -6235,13 +7210,13 @@ paths: - name: path in: path required: true - schema: *ref_35 + schema: *ref_41 - name: parent_job description: >- The parent job that is at the origin and responsible for the execution of this script if any in: query - schema: *ref_46 + schema: *ref_53 - name: job_id description: >- The job id to assign to the created job. if missing, job is chosen @@ -6249,7 +7224,7 @@ paths: queue or as a completed job, the request to create one will fail (Bad Request) in: query - schema: *ref_47 + schema: *ref_54 - name: include_header description: > List of headers's keys (separated with ',') whove value are added to @@ -6258,14 +7233,14 @@ paths: Header's key lowercased and '-'' replaced to '_' such that 'Content-Type' becomes the 'content_type' arg key in: query - schema: &ref_50 + schema: &ref_57 type: string - name: queue_limit description: > The maximum size of the queue for which the request would get rejected if that job would push it above that limit in: query - schema: &ref_51 + schema: &ref_58 type: string requestBody: description: script args @@ -6274,7 +7249,7 @@ paths: application/json: schema: type: object - additionalProperties: *ref_18 + additionalProperties: *ref_21 responses: '200': description: job result @@ -6295,23 +7270,23 @@ paths: - name: path in: path required: true - schema: *ref_35 + schema: *ref_41 - name: parent_job description: >- The parent job that is at the origin and responsible for the execution of this script if any in: query - schema: *ref_46 + schema: *ref_53 - name: tag description: Override the tag to use in: query - schema: *ref_48 + schema: *ref_55 - name: cache_ttl description: >- Override the cache time to live (in seconds). Can not be used to disable caching, only override with a new cache ttl in: query - schema: *ref_49 + schema: *ref_56 - name: job_id description: >- The job id to assign to the created job. if missing, job is chosen @@ -6319,7 +7294,7 @@ paths: queue or as a completed job, the request to create one will fail (Bad Request) in: query - schema: *ref_47 + schema: *ref_54 - name: include_header description: > List of headers's keys (separated with ',') whove value are added to @@ -6328,13 +7303,13 @@ paths: Header's key lowercased and '-'' replaced to '_' such that 'Content-Type' becomes the 'content_type' arg key in: query - schema: *ref_50 + schema: *ref_57 - name: queue_limit description: > The maximum size of the queue for which the request would get rejected if that job would push it above that limit in: query - schema: *ref_51 + schema: *ref_58 requestBody: description: script args required: true @@ -6342,7 +7317,7 @@ paths: application/json: schema: type: object - additionalProperties: *ref_18 + additionalProperties: *ref_21 responses: '200': description: job result @@ -6362,23 +7337,23 @@ paths: - name: path in: path required: true - schema: *ref_35 + schema: *ref_41 - name: parent_job description: >- The parent job that is at the origin and responsible for the execution of this script if any in: query - schema: *ref_46 + schema: *ref_53 - name: tag description: Override the tag to use in: query - schema: *ref_48 + schema: *ref_55 - name: cache_ttl description: >- Override the cache time to live (in seconds). Can not be used to disable caching, only override with a new cache ttl in: query - schema: *ref_49 + schema: *ref_56 - name: job_id description: >- The job id to assign to the created job. if missing, job is chosen @@ -6386,7 +7361,7 @@ paths: queue or as a completed job, the request to create one will fail (Bad Request) in: query - schema: *ref_47 + schema: *ref_54 - name: include_header description: > List of headers's keys (separated with ',') whove value are added to @@ -6395,13 +7370,13 @@ paths: Header's key lowercased and '-'' replaced to '_' such that 'Content-Type' becomes the 'content_type' arg key in: query - schema: *ref_50 + schema: *ref_57 - name: queue_limit description: > The maximum size of the queue for which the request would get rejected if that job would push it above that limit in: query - schema: *ref_51 + schema: *ref_58 - name: payload description: > The base64 encoded payload that has been encoded as a JSON. e.g how @@ -6409,7 +7384,7 @@ paths: `encodeURIComponent(btoa(JSON.stringify({a: 2})))` in: query - schema: &ref_94 + schema: &ref_113 type: string responses: '200': @@ -6431,7 +7406,7 @@ paths: - name: path in: path required: true - schema: *ref_35 + schema: *ref_41 - name: include_header description: > List of headers's keys (separated with ',') whove value are added to @@ -6440,13 +7415,13 @@ paths: Header's key lowercased and '-'' replaced to '_' such that 'Content-Type' becomes the 'content_type' arg key in: query - schema: *ref_50 + schema: *ref_57 - name: queue_limit description: > The maximum size of the queue for which the request would get rejected if that job would push it above that limit in: query - schema: *ref_51 + schema: *ref_58 - name: job_id description: >- The job id to assign to the created job. if missing, job is chosen @@ -6454,7 +7429,7 @@ paths: queue or as a completed job, the request to create one will fail (Bad Request) in: query - schema: *ref_47 + schema: *ref_54 requestBody: description: script args required: true @@ -6462,7 +7437,7 @@ paths: application/json: schema: type: object - additionalProperties: *ref_18 + additionalProperties: *ref_21 responses: '200': description: job result @@ -6483,7 +7458,7 @@ paths: - name: path in: path required: true - schema: *ref_35 + schema: *ref_41 - name: include_header description: > List of headers's keys (separated with ',') whove value are added to @@ -6492,13 +7467,13 @@ paths: Header's key lowercased and '-'' replaced to '_' such that 'Content-Type' becomes the 'content_type' arg key in: query - schema: *ref_50 + schema: *ref_57 - name: queue_limit description: > The maximum size of the queue for which the request would get rejected if that job would push it above that limit in: query - schema: *ref_51 + schema: *ref_58 - name: job_id description: >- The job id to assign to the created job. if missing, job is chosen @@ -6506,7 +7481,7 @@ paths: queue or as a completed job, the request to create one will fail (Bad Request) in: query - schema: *ref_47 + schema: *ref_54 requestBody: description: script args required: true @@ -6514,7 +7489,7 @@ paths: application/json: schema: type: object - additionalProperties: *ref_18 + additionalProperties: *ref_21 responses: '200': description: job result @@ -6617,11 +7592,11 @@ paths: - name: order_desc description: order by desc order (default true) in: query - schema: *ref_52 + schema: *ref_59 - name: created_by description: mask to filter exact matching user creator in: query - schema: *ref_53 + schema: *ref_60 - name: path_start description: mask to filter matching starting path in: query @@ -6675,12 +7650,12 @@ paths: type: array items: allOf: - - allOf: &ref_58 + - allOf: &ref_65 - type: object - properties: *ref_54 - required: *ref_55 + properties: *ref_61 + required: *ref_62 - type: object - properties: &ref_190 + properties: &ref_285 workspace_id: type: string path: @@ -6694,7 +7669,7 @@ paths: type: boolean extra_perms: type: object - additionalProperties: &ref_189 + additionalProperties: &ref_284 type: boolean starred: type: boolean @@ -6712,12 +7687,18 @@ paths: type: number visible_to_runner_only: type: boolean - required: &ref_191 + on_behalf_of_email: + type: string + required: &ref_286 - path - edited_by - edited_at - archived - extra_perms + - type: object + properties: + lock_error_logs: + type: string - type: object properties: has_draft: @@ -6736,7 +7717,7 @@ paths: - name: path in: path required: true - schema: *ref_35 + schema: *ref_41 tags: - flow responses: @@ -6748,7 +7729,7 @@ paths: type: array items: type: object - properties: &ref_56 + properties: &ref_63 id: type: integer created_at: @@ -6756,7 +7737,7 @@ paths: format: date-time deployment_msg: type: string - required: &ref_57 + required: &ref_64 - id - created_at /w/{workspace}/flows/get_latest_version/{path}: @@ -6771,7 +7752,7 @@ paths: - name: path in: path required: true - schema: *ref_35 + schema: *ref_41 tags: - flow responses: @@ -6779,11 +7760,42 @@ paths: description: Flow version content: application/json: - required: false schema: type: object - properties: *ref_56 - required: *ref_57 + properties: *ref_63 + required: *ref_64 + /w/{workspace}/flows/list_paths_from_workspace_runnable/{runnable_kind}/{path}: + get: + summary: list flow paths from workspace runnable + operationId: listFlowPathsFromWorkspaceRunnable + tags: + - flow + parameters: + - name: workspace + in: path + required: true + schema: *ref_0 + - name: runnable_kind + in: path + required: true + schema: &ref_73 + type: string + enum: + - script + - flow + - name: path + in: path + required: true + schema: *ref_41 + responses: + '200': + description: list of flow paths + content: + application/json: + schema: + type: array + items: + type: string /w/{workspace}/flows/get/v/{version}/p/{path}: get: summary: get flow version @@ -6793,8 +7805,7 @@ paths: in: path required: true schema: *ref_0 - - type: string - name: version + - name: version in: path required: true schema: @@ -6802,7 +7813,7 @@ paths: - name: path in: path required: true - schema: *ref_35 + schema: *ref_41 tags: - flow responses: @@ -6811,7 +7822,7 @@ paths: content: application/json: schema: - allOf: *ref_58 + allOf: *ref_65 /w/{workspace}/flows/history_update/v/{version}/p/{path}: post: summary: update flow history @@ -6821,8 +7832,7 @@ paths: in: path required: true schema: *ref_0 - - type: string - name: version + - name: version in: path required: true schema: @@ -6830,7 +7840,7 @@ paths: - name: path in: path required: true - schema: *ref_35 + schema: *ref_41 requestBody: description: Flow deployment message required: true @@ -6866,7 +7876,7 @@ paths: - name: path in: path required: true - schema: *ref_35 + schema: *ref_41 - name: with_starred_info in: query schema: @@ -6877,7 +7887,32 @@ paths: content: application/json: schema: - allOf: *ref_58 + allOf: *ref_65 + /w/{workspace}/flows/deployment_status/p/{path}: + get: + summary: get flow deployment status + operationId: getFlowDeploymentStatus + tags: + - flow + parameters: + - name: workspace + in: path + required: true + schema: *ref_0 + - name: path + in: path + required: true + schema: *ref_41 + responses: + '200': + description: flow status + content: + application/json: + schema: + type: object + properties: + lock_error_logs: + type: string /w/{workspace}/flows/get_triggers_count/{path}: get: summary: get triggers count of flow @@ -6892,7 +7927,7 @@ paths: - name: path in: path required: true - schema: *ref_35 + schema: *ref_41 responses: '200': description: triggers count @@ -6900,7 +7935,7 @@ paths: application/json: schema: type: object - properties: *ref_59 + properties: *ref_66 /w/{workspace}/flows/list_tokens/{path}: get: summary: get tokens with flow scope @@ -6915,7 +7950,7 @@ paths: - name: path in: path required: true - schema: *ref_35 + schema: *ref_41 responses: '200': description: tokens list @@ -6925,8 +7960,8 @@ paths: type: array items: type: object - properties: *ref_40 - required: *ref_41 + properties: *ref_47 + required: *ref_48 /w/{workspace}/flows/toggle_workspace_error_handler/{path}: post: summary: Toggle ON and OFF the workspace error handler for a given flow @@ -6941,7 +7976,7 @@ paths: - name: path in: path required: true - schema: *ref_35 + schema: *ref_41 requestBody: description: Workspace error handler enabled required: true @@ -6973,7 +8008,7 @@ paths: - name: path in: path required: true - schema: *ref_35 + schema: *ref_41 responses: '200': description: flow details with draft @@ -6981,11 +8016,11 @@ paths: application/json: schema: allOf: - - allOf: *ref_58 + - allOf: *ref_65 - type: object properties: draft: - allOf: *ref_58 + allOf: *ref_65 /w/{workspace}/flows/exists/{path}: get: summary: exists flow by path @@ -7000,7 +8035,7 @@ paths: - name: path in: path required: true - schema: *ref_35 + schema: *ref_41 responses: '200': description: flow details @@ -7026,10 +8061,10 @@ paths: application/json: schema: allOf: - - allOf: &ref_60 + - allOf: &ref_67 - type: object - properties: *ref_54 - required: *ref_55 + properties: *ref_61 + required: *ref_62 - type: object properties: path: @@ -7046,6 +8081,8 @@ paths: type: number visible_to_runner_only: type: boolean + on_behalf_of_email: + type: string required: - path - type: object @@ -7075,7 +8112,7 @@ paths: - name: path in: path required: true - schema: *ref_35 + schema: *ref_41 requestBody: description: Partially filled flow required: true @@ -7083,7 +8120,7 @@ paths: application/json: schema: allOf: - - allOf: *ref_60 + - allOf: *ref_67 - type: object properties: deployment_message: @@ -7109,7 +8146,7 @@ paths: - name: path in: path required: true - schema: *ref_35 + schema: *ref_41 requestBody: description: archiveFlow required: true @@ -7141,7 +8178,12 @@ paths: - name: path in: path required: true - schema: *ref_35 + schema: *ref_41 + - name: keep_captures + description: keep captures + in: query + schema: + type: boolean responses: '200': description: flow delete @@ -7171,11 +8213,11 @@ paths: - name: order_desc description: order by desc order (default true) in: query - schema: *ref_52 + schema: *ref_59 - name: created_by description: mask to filter exact matching user creator in: query - schema: *ref_53 + schema: *ref_60 - name: path_start description: mask to filter matching starting path in: query @@ -7202,7 +8244,7 @@ paths: type: array items: type: object - properties: &ref_197 + properties: &ref_292 workspace_id: type: string path: @@ -7220,7 +8262,7 @@ paths: edited_at: type: string format: date-time - required: &ref_198 + required: &ref_293 - workspace_id - path - summary @@ -7241,7 +8283,7 @@ paths: - name: path in: path required: true - schema: *ref_23 + schema: *ref_26 responses: '200': description: app exists @@ -7263,12 +8305,12 @@ paths: - name: version in: path required: true - schema: &ref_125 + schema: &ref_186 type: number - name: path in: path required: true - schema: *ref_35 + schema: *ref_41 responses: '200': description: app details @@ -7325,11 +8367,11 @@ paths: - name: order_desc description: order by desc order (default true) in: query - schema: *ref_52 + schema: *ref_59 - name: created_by description: mask to filter exact matching user creator in: query - schema: *ref_53 + schema: *ref_60 - name: path_start description: mask to filter matching starting path in: query @@ -7370,7 +8412,7 @@ paths: type: array items: type: object - properties: &ref_195 + properties: &ref_290 id: type: integer workspace_id: @@ -7396,7 +8438,9 @@ paths: - viewer - publisher - anonymous - required: &ref_196 + raw_app: + type: boolean + required: &ref_291 - id - workspace_id - path @@ -7431,33 +8475,13 @@ paths: type: string policy: type: object - properties: &ref_61 - triggerables: - type: object - additionalProperties: - type: object - triggerables_v2: - type: object - additionalProperties: - type: object - s3_inputs: - type: array - items: - type: object - execution_mode: - type: string - enum: - - viewer - - publisher - - anonymous - on_behalf_of: - type: string - on_behalf_of_email: - type: string + properties: *ref_68 draft_only: type: boolean deployment_message: type: string + custom_path: + type: string required: - path - value @@ -7470,6 +8494,58 @@ paths: text/plain: schema: type: string + /w/{workspace}/apps/create_raw: + post: + summary: create app raw + operationId: createAppRaw + tags: + - app + parameters: + - name: workspace + in: path + required: true + schema: *ref_0 + requestBody: + description: new app + required: true + content: + multipart/form-data: + schema: + type: object + properties: + app: + type: object + properties: + path: + type: string + value: {} + summary: + type: string + policy: + type: object + properties: *ref_68 + draft_only: + type: boolean + deployment_message: + type: string + custom_path: + type: string + required: + - path + - value + - summary + - policy + js: + type: string + css: + type: string + responses: + '201': + description: app created + content: + text/plain: + schema: + type: string /w/{workspace}/apps/exists/{path}: get: summary: does an app exisst at path @@ -7484,7 +8560,7 @@ paths: - name: path in: path required: true - schema: *ref_23 + schema: *ref_26 responses: '200': description: app exists @@ -7506,7 +8582,7 @@ paths: - name: path in: path required: true - schema: *ref_35 + schema: *ref_41 - name: with_starred_info in: query schema: @@ -7518,51 +8594,32 @@ paths: application/json: schema: type: object - properties: &ref_62 - id: - type: integer - workspace_id: - type: string - path: - type: string - summary: - type: string - versions: - type: array - items: - type: integer - created_by: - type: string - created_at: - type: string - format: date-time - value: - type: object - policy: - type: object - properties: *ref_61 - execution_mode: - type: string - enum: - - viewer - - publisher - - anonymous - extra_perms: - type: object - additionalProperties: - type: boolean - required: &ref_63 - - id - - workspace_id - - path - - summary - - versions - - created_by - - created_at - - value - - policy - - execution_mode - - extra_perms + properties: *ref_69 + required: *ref_70 + /w/{workspace}/apps/get/lite/{path}: + get: + summary: get app lite by path + operationId: getAppLiteByPath + tags: + - app + parameters: + - name: workspace + in: path + required: true + schema: *ref_0 + - name: path + in: path + required: true + schema: *ref_41 + responses: + '200': + description: app lite details + content: + application/json: + schema: + type: object + properties: *ref_69 + required: *ref_70 /w/{workspace}/apps/get/draft/{path}: get: summary: get app by path with draft @@ -7577,17 +8634,17 @@ paths: - name: path in: path required: true - schema: *ref_35 + schema: *ref_41 responses: '200': description: app details with draft content: application/json: schema: - allOf: &ref_199 + allOf: &ref_294 - type: object - properties: *ref_62 - required: *ref_63 + properties: *ref_69 + required: *ref_70 - type: object properties: draft_only: @@ -7607,7 +8664,7 @@ paths: - name: path in: path required: true - schema: *ref_35 + schema: *ref_41 responses: '200': description: app history @@ -7617,12 +8674,12 @@ paths: type: array items: type: object - properties: &ref_64 + properties: &ref_71 version: type: integer deployment_msg: type: string - required: &ref_65 + required: &ref_72 - version /w/{workspace}/apps/get_latest_version/{path}: get: @@ -7636,7 +8693,7 @@ paths: - name: path in: path required: true - schema: *ref_35 + schema: *ref_41 tags: - app responses: @@ -7644,11 +8701,38 @@ paths: description: App version content: application/json: - required: false schema: type: object - properties: *ref_64 - required: *ref_65 + properties: *ref_71 + required: *ref_72 + /w/{workspace}/apps/list_paths_from_workspace_runnable/{runnable_kind}/{path}: + get: + summary: list app paths from workspace runnable + operationId: listAppPathsFromWorkspaceRunnable + tags: + - app + parameters: + - name: workspace + in: path + required: true + schema: *ref_0 + - name: runnable_kind + in: path + required: true + schema: *ref_73 + - name: path + in: path + required: true + schema: *ref_41 + responses: + '200': + description: list of app paths + content: + application/json: + schema: + type: array + items: + type: string /w/{workspace}/apps/history_update/a/{id}/v/{version}: post: summary: update app history @@ -7663,11 +8747,11 @@ paths: - name: id in: path required: true - schema: *ref_30 + schema: *ref_34 - name: version in: path required: true - schema: &ref_127 + schema: &ref_188 type: integer requestBody: description: App deployment message @@ -7700,7 +8784,7 @@ paths: - name: path in: path required: true - schema: *ref_23 + schema: *ref_26 responses: '200': description: app details @@ -7708,8 +8792,8 @@ paths: application/json: schema: type: object - properties: *ref_62 - required: *ref_63 + properties: *ref_69 + required: *ref_70 /w/{workspace}/apps_u/public_resource/{path}: get: summary: get public resource @@ -7724,7 +8808,7 @@ paths: - name: path in: path required: true - schema: *ref_23 + schema: *ref_26 responses: '200': description: resource value @@ -7745,7 +8829,7 @@ paths: - name: path in: path required: true - schema: *ref_23 + schema: *ref_26 responses: '200': description: app secret @@ -7767,7 +8851,7 @@ paths: - name: id in: path required: true - schema: *ref_30 + schema: *ref_34 responses: '200': description: app details @@ -7775,8 +8859,8 @@ paths: application/json: schema: type: object - properties: *ref_62 - required: *ref_63 + properties: *ref_69 + required: *ref_70 /w/{workspace}/raw_apps/create: post: summary: create raw app @@ -7827,7 +8911,7 @@ paths: - name: path in: path required: true - schema: *ref_35 + schema: *ref_41 requestBody: description: updateraw app required: true @@ -7863,7 +8947,7 @@ paths: - name: path in: path required: true - schema: *ref_23 + schema: *ref_26 responses: '200': description: app deleted @@ -7885,7 +8969,7 @@ paths: - name: path in: path required: true - schema: *ref_23 + schema: *ref_26 responses: '200': description: app deleted @@ -7907,7 +8991,7 @@ paths: - name: path in: path required: true - schema: *ref_35 + schema: *ref_41 requestBody: description: update app required: true @@ -7923,9 +9007,11 @@ paths: value: {} policy: type: object - properties: *ref_61 + properties: *ref_68 deployment_message: type: string + custom_path: + type: string responses: '200': description: app updated @@ -7933,6 +9019,124 @@ paths: text/plain: schema: type: string + /w/{workspace}/apps/update_raw/{path}: + post: + summary: update app + operationId: updateAppRaw + tags: + - app + parameters: + - name: workspace + in: path + required: true + schema: *ref_0 + - name: path + in: path + required: true + schema: *ref_41 + requestBody: + description: update app + required: true + content: + multipart/form-data: + schema: + type: object + properties: + app: + type: object + properties: + path: + type: string + summary: + type: string + value: {} + policy: + type: object + properties: *ref_68 + deployment_message: + type: string + custom_path: + type: string + js: + type: string + css: + type: string + responses: + '200': + description: app updated + content: + text/plain: + schema: + type: string + /w/{workspace}/apps/custom_path_exists/{custom_path}: + get: + summary: check if custom path exists + operationId: customPathExists + tags: + - app + parameters: + - name: workspace + in: path + required: true + schema: *ref_0 + - name: custom_path + in: path + required: true + schema: *ref_74 + responses: + '200': + description: custom path exists + content: + application/json: + schema: + type: boolean + /w/{workspace}/apps/sign_s3_objects: + post: + summary: sign s3 objects, to be used by anonymous users in public apps + operationId: signS3Objects + tags: + - app + parameters: + - name: workspace + in: path + required: true + schema: *ref_0 + requestBody: + description: s3 objects to sign + required: true + content: + application/json: + schema: + type: object + properties: + s3_objects: + type: array + items: + type: object + properties: &ref_75 + s3: + type: string + filename: + type: string + storage: + type: string + presigned: + type: string + required: &ref_76 + - s3 + required: + - s3_objects + responses: + '200': + description: signed s3 objects + content: + application/json: + schema: + type: array + items: + type: object + properties: *ref_75 + required: *ref_76 /w/{workspace}/apps_u/execute_component/{path}: post: summary: executeComponent @@ -7947,7 +9151,7 @@ paths: - name: path in: path required: true - schema: *ref_35 + schema: *ref_41 requestBody: description: update app required: true @@ -7960,6 +9164,8 @@ paths: type: string path: type: string + version: + type: integer args: {} raw_code: type: object @@ -7977,6 +9183,8 @@ paths: required: - content - language + id: + type: integer force_viewer_static_fields: type: object force_viewer_one_of_fields: @@ -7995,6 +9203,99 @@ paths: text/plain: schema: type: string + /w/{workspace}/apps_u/upload_s3_file/{path}: + post: + summary: upload s3 file from app + operationId: uploadS3FileFromApp + tags: + - app + parameters: + - name: workspace + in: path + required: true + schema: *ref_0 + - name: path + in: path + required: true + schema: *ref_26 + - name: file_key + in: query + required: false + schema: + type: string + - name: file_extension + in: query + required: false + schema: + type: string + - name: s3_resource_path + in: query + required: false + schema: + type: string + - name: resource_type + in: query + required: false + schema: + type: string + - name: storage + in: query + schema: + type: string + - name: content_type + in: query + schema: + type: string + - name: content_disposition + in: query + schema: + type: string + requestBody: + description: File content + required: true + content: + application/octet-stream: + schema: + type: string + format: binary + responses: + '200': + description: file uploaded + content: + application/json: + schema: + type: object + properties: + file_key: + type: string + delete_token: + type: string + required: + - file_key + - delete_token + /w/{workspace}/apps_u/delete_s3_file: + delete: + summary: delete s3 file from app + operationId: deleteS3FileFromApp + tags: + - app + parameters: + - name: workspace + in: path + required: true + schema: *ref_0 + - name: delete_token + in: query + required: true + schema: + type: string + responses: + '200': + description: file deleted + content: + text/plain: + schema: + type: string /w/{workspace}/jobs/run/f/{path}: post: summary: run flow by path @@ -8009,7 +9310,7 @@ paths: - name: path in: path required: true - schema: *ref_35 + schema: *ref_41 - name: scheduled_for description: when to schedule this job (leave empty for immediate run) in: query @@ -8031,11 +9332,11 @@ paths: The parent job that is at the origin and responsible for the execution of this script if any in: query - schema: *ref_46 + schema: *ref_53 - name: tag description: Override the tag to use in: query - schema: *ref_48 + schema: *ref_55 - name: job_id description: >- The job id to assign to the created job. if missing, job is chosen @@ -8043,7 +9344,7 @@ paths: queue or as a completed job, the request to create one will fail (Bad Request) in: query - schema: *ref_47 + schema: *ref_54 - name: include_header description: > List of headers's keys (separated with ',') whove value are added to @@ -8052,7 +9353,7 @@ paths: Header's key lowercased and '-'' replaced to '_' such that 'Content-Type' becomes the 'content_type' arg key in: query - schema: *ref_50 + schema: *ref_57 - name: invisible_to_owner description: make the run invisible to the the flow owner (default false) in: query @@ -8065,7 +9366,7 @@ paths: application/json: schema: type: object - additionalProperties: *ref_18 + additionalProperties: *ref_21 responses: '201': description: job created @@ -8074,6 +9375,66 @@ paths: schema: type: string format: uuid + /w/{workspace}/jobs/run/batch_rerun_jobs: + post: + summary: re-run multiple jobs + operationId: batchReRunJobs + tags: + - job + parameters: + - name: workspace + in: path + required: true + schema: *ref_0 + requestBody: + description: list of job ids to re run and arg tranforms + required: true + content: + application/json: + schema: + type: object + required: + - job_ids + - script_options_by_path + - flow_options_by_path + properties: + job_ids: + type: array + items: + type: string + script_options_by_path: + type: object + additionalProperties: + type: object + properties: + input_transforms: + type: object + additionalProperties: + allOf: &ref_77 + - oneOf: *ref_35 + discriminator: *ref_36 + use_latest_version: + type: boolean + flow_options_by_path: + type: object + additionalProperties: + type: object + properties: + input_transforms: + type: object + additionalProperties: + allOf: *ref_77 + use_latest_version: + type: boolean + responses: + '201': + description: >- + stream of created job uuids separated by \n. Lines may start with + 'Error:' + content: + text/event-stream: + schema: + type: string /w/{workspace}/jobs/restart/f/{id}/from/{step_id}/{branch_or_iteration_n}: post: summary: restart a completed flow at a given step @@ -8088,7 +9449,7 @@ paths: - name: id in: path required: true - schema: &ref_91 + schema: &ref_110 type: string format: uuid - name: step_id @@ -8121,11 +9482,11 @@ paths: The parent job that is at the origin and responsible for the execution of this script if any in: query - schema: *ref_46 + schema: *ref_53 - name: tag description: Override the tag to use in: query - schema: *ref_48 + schema: *ref_55 - name: job_id description: >- The job id to assign to the created job. if missing, job is chosen @@ -8133,7 +9494,7 @@ paths: queue or as a completed job, the request to create one will fail (Bad Request) in: query - schema: *ref_47 + schema: *ref_54 - name: include_header description: > List of headers's keys (separated with ',') whove value are added to @@ -8142,7 +9503,7 @@ paths: Header's key lowercased and '-'' replaced to '_' such that 'Content-Type' becomes the 'content_type' arg key in: query - schema: *ref_50 + schema: *ref_57 - name: invisible_to_owner description: make the run invisible to the the flow owner (default false) in: query @@ -8155,7 +9516,7 @@ paths: application/json: schema: type: object - additionalProperties: *ref_18 + additionalProperties: *ref_21 responses: '201': description: job created @@ -8178,7 +9539,7 @@ paths: - name: hash in: path required: true - schema: *ref_39 + schema: *ref_46 - name: scheduled_for description: when to schedule this job (leave empty for immediate run) in: query @@ -8200,17 +9561,17 @@ paths: The parent job that is at the origin and responsible for the execution of this script if any in: query - schema: *ref_46 + schema: *ref_53 - name: tag description: Override the tag to use in: query - schema: *ref_48 + schema: *ref_55 - name: cache_ttl description: >- Override the cache time to live (in seconds). Can not be used to disable caching, only override with a new cache ttl in: query - schema: *ref_49 + schema: *ref_56 - name: job_id description: >- The job id to assign to the created job. if missing, job is chosen @@ -8218,7 +9579,7 @@ paths: queue or as a completed job, the request to create one will fail (Bad Request) in: query - schema: *ref_47 + schema: *ref_54 - name: include_header description: > List of headers's keys (separated with ',') whove value are added to @@ -8227,7 +9588,7 @@ paths: Header's key lowercased and '-'' replaced to '_' such that 'Content-Type' becomes the 'content_type' arg key in: query - schema: *ref_50 + schema: *ref_57 - name: invisible_to_owner description: make the run invisible to the the script owner (default false) in: query @@ -8267,7 +9628,7 @@ paths: Header's key lowercased and '-'' replaced to '_' such that 'Content-Type' becomes the 'content_type' arg key in: query - schema: *ref_50 + schema: *ref_57 - name: invisible_to_owner description: make the run invisible to the the script owner (default false) in: query @@ -8280,7 +9641,7 @@ paths: queue or as a completed job, the request to create one will fail (Bad Request) in: query - schema: *ref_47 + schema: *ref_54 requestBody: description: preview required: true @@ -8288,33 +9649,19 @@ paths: application/json: schema: type: object - properties: &ref_156 + properties: &ref_221 content: type: string path: type: string + script_hash: + type: string args: type: object - additionalProperties: *ref_18 + additionalProperties: *ref_21 language: type: string - enum: - - python3 - - deno - - go - - bash - - powershell - - postgresql - - mysql - - bigquery - - snowflake - - mssql - - graphql - - nativets - - bun - - php - - rust - - ansible + enum: *ref_43 tag: type: string kind: @@ -8327,7 +9674,7 @@ paths: type: boolean lock: type: string - required: &ref_157 + required: &ref_222 - args responses: '201': @@ -8365,11 +9712,11 @@ paths: application/json: schema: type: object - properties: &ref_158 + properties: &ref_223 args: type: object - additionalProperties: *ref_18 - required: &ref_159 + additionalProperties: *ref_21 + required: &ref_224 - args responses: '201': @@ -8402,31 +9749,15 @@ paths: type: array items: type: object - properties: &ref_217 + properties: &ref_312 raw_code: type: string path: type: string language: type: string - enum: - - python3 - - deno - - go - - bash - - powershell - - postgresql - - mysql - - bigquery - - snowflake - - mssql - - graphql - - nativets - - bun - - php - - rust - - ansible - required: &ref_218 + enum: *ref_43 + required: &ref_313 - raw_code - path - language @@ -8466,7 +9797,7 @@ paths: Header's key lowercased and '-'' replaced to '_' such that 'Content-Type' becomes the 'content_type' arg key in: query - schema: *ref_50 + schema: *ref_57 - name: invisible_to_owner description: make the run invisible to the the script owner (default false) in: query @@ -8479,7 +9810,7 @@ paths: queue or as a completed job, the request to create one will fail (Bad Request) in: query - schema: *ref_47 + schema: *ref_54 requestBody: description: preview required: true @@ -8487,21 +9818,21 @@ paths: application/json: schema: type: object - properties: &ref_192 + properties: &ref_287 value: type: object - properties: *ref_66 - required: *ref_67 + properties: *ref_78 + required: *ref_79 path: type: string args: type: object - additionalProperties: *ref_18 + additionalProperties: *ref_21 tag: type: string restarted_from: type: object - properties: &ref_194 + properties: &ref_289 flow_job_id: type: string format: uuid @@ -8509,7 +9840,7 @@ paths: type: string branch_or_iteration_n: type: integer - required: &ref_193 + required: &ref_288 - value - content - args @@ -8535,94 +9866,104 @@ paths: - name: order_desc description: order by desc order (default true) in: query - schema: *ref_52 + schema: *ref_59 - name: created_by description: mask to filter exact matching user creator in: query - schema: *ref_53 + schema: *ref_60 - name: parent_job description: >- The parent job that is at the origin and responsible for the execution of this script if any in: query - schema: *ref_46 + schema: *ref_53 + - name: worker + description: worker this job was ran on + in: query + schema: &ref_82 + type: string - name: script_path_exact description: mask to filter exact matching path in: query - schema: &ref_70 + schema: &ref_83 type: string - name: script_path_start description: mask to filter matching starting path in: query - schema: &ref_71 + schema: &ref_84 type: string - name: schedule_path description: mask to filter by schedule path in: query - schema: &ref_72 + schema: &ref_85 type: string - name: script_hash description: mask to filter exact matching path in: query - schema: &ref_73 + schema: &ref_86 type: string - name: started_before description: filter on started before (inclusive) timestamp in: query - schema: &ref_74 + schema: &ref_87 type: string format: date-time - name: started_after description: filter on started after (exclusive) timestamp in: query - schema: &ref_75 + schema: &ref_88 type: string format: date-time - name: success description: filter on successful jobs in: query - schema: &ref_76 + schema: &ref_96 type: boolean - name: scheduled_for_before_now description: filter on jobs scheduled_for before now (hence waitinf for a worker) in: query - schema: &ref_77 + schema: &ref_90 type: boolean - name: job_kinds description: >- filter on job kind (values 'preview', 'script', 'dependencies', 'flow') separated by, in: query - schema: &ref_78 + schema: &ref_91 type: string - name: suspended description: filter on suspended jobs in: query - schema: &ref_79 + schema: &ref_92 type: boolean - name: running description: filter on running jobs in: query - schema: &ref_80 + schema: &ref_89 type: boolean - name: args description: >- filter on jobs containing those args as a json subset (@> in postgres) in: query - schema: &ref_81 + schema: &ref_93 type: string - name: result description: >- filter on jobs containing those result as a json subset (@> in postgres) in: query - schema: &ref_82 + schema: &ref_95 type: string + - name: allow_wildcards + description: allow wildcards (*) in the filter of label, tag, worker + in: query + schema: &ref_97 + type: boolean - name: tag description: filter on jobs with a given tag/worker group in: query - schema: &ref_83 + schema: &ref_94 type: string - name: page description: which page to return (start at 1, default 1) @@ -8653,7 +9994,7 @@ paths: type: array items: type: object - properties: &ref_89 + properties: &ref_108 workspace_id: type: string id: @@ -8681,7 +10022,7 @@ paths: type: string args: type: object - additionalProperties: *ref_18 + additionalProperties: *ref_21 logs: type: string raw_code: @@ -8717,20 +10058,20 @@ paths: permissioned_as: type: string description: > - The user (u/userfoo) or group (g/groupfoo) whom + The user (u/userfoo) or group (g/groupfoo) whom the execution of this script will be permissioned_as and by extension its DT_TOKEN. flow_status: type: object - properties: &ref_84 + properties: &ref_99 step: type: integer modules: type: array items: type: object - properties: &ref_68 + properties: &ref_80 type: type: string enum: @@ -8807,20 +10148,20 @@ paths: format: uuid skipped: type: boolean - required: &ref_69 + required: &ref_81 - type user_states: additionalProperties: true preprocessor_module: allOf: - type: object - properties: *ref_68 - required: *ref_69 + properties: *ref_80 + required: *ref_81 failure_module: allOf: - type: object - properties: *ref_68 - required: *ref_69 + properties: *ref_80 + required: *ref_81 - type: object properties: parent_module: @@ -8835,35 +10176,19 @@ paths: items: type: string format: uuid - required: &ref_85 + required: &ref_100 - step - modules - failure_module raw_flow: type: object - properties: *ref_66 - required: *ref_67 + properties: *ref_78 + required: *ref_79 is_flow_step: type: boolean language: type: string - enum: - - python3 - - deno - - go - - bash - - powershell - - postgresql - - mysql - - bigquery - - snowflake - - mssql - - graphql - - nativets - - bun - - php - - rust - - ansible + enum: *ref_43 email: type: string visible_to_owner: @@ -8880,7 +10205,11 @@ paths: type: number suspend: type: number - required: &ref_90 + preprocessed: + type: boolean + worker: + type: string + required: &ref_109 - id - running - canceled @@ -8945,10 +10274,221 @@ paths: type: integer required: - database_length - /w/{workspace}/jobs/queue/list_filtered_uuids: + /w/{workspace}/jobs/completed/count_jobs: + get: + summary: count number of completed jobs with filter + operationId: countCompletedJobs + tags: + - job + parameters: + - name: workspace + in: path + required: true + schema: *ref_0 + - name: completed_after_s_ago + in: query + schema: + type: integer + - name: success + in: query + schema: + type: boolean + - name: tags + in: query + schema: + type: string + - name: all_workspaces + in: query + schema: + type: boolean + responses: + '200': + description: Count of completed jobs + content: + application/json: + schema: + type: integer + /w/{workspace}/jobs/list_filtered_uuids: get: summary: get the ids of all jobs matching the given filters - operationId: listFilteredUuids + operationId: listFilteredJobsUuids + tags: + - job + parameters: + - name: workspace + in: path + required: true + schema: *ref_0 + - name: created_by + description: mask to filter exact matching user creator + in: query + schema: *ref_60 + - name: label + description: >- + mask to filter exact matching job's label (job labels are completed + jobs with as a result an object containing a string in the array at + key 'wm_labels') + in: query + schema: &ref_98 + type: string + - name: worker + description: worker this job was ran on + in: query + schema: *ref_82 + - name: parent_job + description: >- + The parent job that is at the origin and responsible for the + execution of this script if any + in: query + schema: *ref_53 + - name: script_path_exact + description: mask to filter exact matching path + in: query + schema: *ref_83 + - name: script_path_start + description: mask to filter matching starting path + in: query + schema: *ref_84 + - name: schedule_path + description: mask to filter by schedule path + in: query + schema: *ref_85 + - name: script_hash + description: mask to filter exact matching path + in: query + schema: *ref_86 + - name: started_before + description: filter on started before (inclusive) timestamp + in: query + schema: *ref_87 + - name: started_after + description: filter on started after (exclusive) timestamp + in: query + schema: *ref_88 + - name: created_before + description: filter on created before (inclusive) timestamp + in: query + schema: &ref_101 + type: string + format: date-time + - name: created_after + description: filter on created after (exclusive) timestamp + in: query + schema: &ref_102 + type: string + format: date-time + - name: created_or_started_before + description: >- + filter on created_at for non non started job and started_at + otherwise before (inclusive) timestamp + in: query + schema: &ref_103 + type: string + format: date-time + - name: running + description: filter on running jobs + in: query + schema: *ref_89 + - name: scheduled_for_before_now + description: filter on jobs scheduled_for before now (hence waitinf for a worker) + in: query + schema: *ref_90 + - name: created_or_started_after + description: >- + filter on created_at for non non started job and started_at + otherwise after (exclusive) timestamp + in: query + schema: &ref_104 + type: string + format: date-time + - name: created_or_started_after_completed_jobs + description: >- + filter on created_at for non non started job and started_at + otherwise after (exclusive) timestamp but only for the completed + jobs + in: query + schema: &ref_105 + type: string + format: date-time + - name: job_kinds + description: >- + filter on job kind (values 'preview', 'script', 'dependencies', + 'flow') separated by, + in: query + schema: *ref_91 + - name: suspended + description: filter on suspended jobs + in: query + schema: *ref_92 + - name: args + description: >- + filter on jobs containing those args as a json subset (@> in + postgres) + in: query + schema: *ref_93 + - name: tag + description: filter on jobs with a given tag/worker group + in: query + schema: *ref_94 + - name: result + description: >- + filter on jobs containing those result as a json subset (@> in + postgres) + in: query + schema: *ref_95 + - name: page + description: which page to return (start at 1, default 1) + in: query + schema: *ref_5 + - name: per_page + description: number of items to return for a given page (default 30, max 100) + in: query + schema: *ref_6 + - name: is_skipped + description: is the job skipped + in: query + schema: + type: boolean + - name: is_flow_step + description: is the job a flow step + in: query + schema: + type: boolean + - name: has_null_parent + description: has null parent + in: query + schema: + type: boolean + - name: success + description: filter on successful jobs + in: query + schema: + type: boolean + - name: all_workspaces + description: >- + get jobs from all workspaces (only valid if request come from the + `admins` workspace) + in: query + schema: + type: boolean + - name: is_not_schedule + description: is not a scheduled job + in: query + schema: + type: boolean + responses: + '200': + description: uuids of jobs + content: + application/json: + schema: + type: array + items: + type: string + /w/{workspace}/jobs/queue/list_filtered_uuids: + get: + summary: get the ids of all queued jobs matching the given filters + operationId: listFilteredQueueUuids tags: - job parameters: @@ -8959,79 +10499,83 @@ paths: - name: order_desc description: order by desc order (default true) in: query - schema: *ref_52 + schema: *ref_59 - name: created_by description: mask to filter exact matching user creator in: query - schema: *ref_53 + schema: *ref_60 - name: parent_job description: >- The parent job that is at the origin and responsible for the execution of this script if any in: query - schema: *ref_46 + schema: *ref_53 - name: script_path_exact description: mask to filter exact matching path in: query - schema: *ref_70 + schema: *ref_83 - name: script_path_start description: mask to filter matching starting path in: query - schema: *ref_71 + schema: *ref_84 - name: schedule_path description: mask to filter by schedule path in: query - schema: *ref_72 + schema: *ref_85 - name: script_hash description: mask to filter exact matching path in: query - schema: *ref_73 + schema: *ref_86 - name: started_before description: filter on started before (inclusive) timestamp in: query - schema: *ref_74 + schema: *ref_87 - name: started_after description: filter on started after (exclusive) timestamp in: query - schema: *ref_75 + schema: *ref_88 - name: success description: filter on successful jobs in: query - schema: *ref_76 + schema: *ref_96 - name: scheduled_for_before_now description: filter on jobs scheduled_for before now (hence waitinf for a worker) in: query - schema: *ref_77 + schema: *ref_90 - name: job_kinds description: >- filter on job kind (values 'preview', 'script', 'dependencies', 'flow') separated by, in: query - schema: *ref_78 + schema: *ref_91 - name: suspended description: filter on suspended jobs in: query - schema: *ref_79 + schema: *ref_92 - name: running description: filter on running jobs in: query - schema: *ref_80 + schema: *ref_89 - name: args description: >- filter on jobs containing those args as a json subset (@> in postgres) in: query - schema: *ref_81 + schema: *ref_93 - name: result description: >- filter on jobs containing those result as a json subset (@> in postgres) in: query - schema: *ref_82 + schema: *ref_95 + - name: allow_wildcards + description: allow wildcards (*) in the filter of label, tag, worker + in: query + schema: *ref_97 - name: tag description: filter on jobs with a given tag/worker group in: query - schema: *ref_83 + schema: *ref_94 - name: page description: which page to return (start at 1, default 1) in: query @@ -9109,75 +10653,82 @@ paths: - name: order_desc description: order by desc order (default true) in: query - schema: *ref_52 + schema: *ref_59 - name: created_by description: mask to filter exact matching user creator in: query - schema: *ref_53 + schema: *ref_60 - name: label description: >- mask to filter exact matching job's label (job labels are completed jobs with as a result an object containing a string in the array at key 'wm_labels') in: query - schema: &ref_86 - type: string + schema: *ref_98 + - name: worker + description: worker this job was ran on + in: query + schema: *ref_82 - name: parent_job description: >- The parent job that is at the origin and responsible for the execution of this script if any in: query - schema: *ref_46 + schema: *ref_53 - name: script_path_exact description: mask to filter exact matching path in: query - schema: *ref_70 + schema: *ref_83 - name: script_path_start description: mask to filter matching starting path in: query - schema: *ref_71 + schema: *ref_84 - name: schedule_path description: mask to filter by schedule path in: query - schema: *ref_72 + schema: *ref_85 - name: script_hash description: mask to filter exact matching path in: query - schema: *ref_73 + schema: *ref_86 - name: started_before description: filter on started before (inclusive) timestamp in: query - schema: *ref_74 + schema: *ref_87 - name: started_after description: filter on started after (exclusive) timestamp in: query - schema: *ref_75 + schema: *ref_88 - name: success description: filter on successful jobs in: query - schema: *ref_76 + schema: *ref_96 - name: job_kinds description: >- filter on job kind (values 'preview', 'script', 'dependencies', 'flow') separated by, in: query - schema: *ref_78 + schema: *ref_91 - name: args description: >- filter on jobs containing those args as a json subset (@> in postgres) in: query - schema: *ref_81 + schema: *ref_93 - name: result description: >- filter on jobs containing those result as a json subset (@> in postgres) in: query - schema: *ref_82 + schema: *ref_95 + - name: allow_wildcards + description: allow wildcards (*) in the filter of label, tag, worker + in: query + schema: *ref_97 - name: tag description: filter on jobs with a given tag/worker group in: query - schema: *ref_83 + schema: *ref_94 - name: page description: which page to return (start at 1, default 1) in: query @@ -9215,7 +10766,7 @@ paths: type: array items: type: object - properties: &ref_87 + properties: &ref_106 workspace_id: type: string id: @@ -9242,7 +10793,7 @@ paths: type: string args: type: object - additionalProperties: *ref_18 + additionalProperties: *ref_21 result: {} logs: type: string @@ -9278,39 +10829,23 @@ paths: permissioned_as: type: string description: > - The user (u/userfoo) or group (g/groupfoo) whom + The user (u/userfoo) or group (g/groupfoo) whom the execution of this script will be permissioned_as and by extension its DT_TOKEN. flow_status: type: object - properties: *ref_84 - required: *ref_85 + properties: *ref_99 + required: *ref_100 raw_flow: type: object - properties: *ref_66 - required: *ref_67 + properties: *ref_78 + required: *ref_79 is_flow_step: type: boolean language: type: string - enum: - - python3 - - deno - - go - - bash - - powershell - - postgresql - - mysql - - bigquery - - snowflake - - mssql - - graphql - - nativets - - bun - - php - - rust - - ansible + enum: *ref_43 is_skipped: type: boolean email: @@ -9331,7 +10866,11 @@ paths: type: number aggregate_wait_time_ms: type: number - required: &ref_88 + preprocessed: + type: boolean + worker: + type: string + required: &ref_107 - id - created_by - duration_ms @@ -9360,115 +10899,113 @@ paths: - name: created_by description: mask to filter exact matching user creator in: query - schema: *ref_53 + schema: *ref_60 - name: label description: >- mask to filter exact matching job's label (job labels are completed jobs with as a result an object containing a string in the array at key 'wm_labels') in: query - schema: *ref_86 + schema: *ref_98 + - name: worker + description: worker this job was ran on + in: query + schema: *ref_82 - name: parent_job description: >- The parent job that is at the origin and responsible for the execution of this script if any in: query - schema: *ref_46 + schema: *ref_53 - name: script_path_exact description: mask to filter exact matching path in: query - schema: *ref_70 + schema: *ref_83 - name: script_path_start description: mask to filter matching starting path in: query - schema: *ref_71 + schema: *ref_84 - name: schedule_path description: mask to filter by schedule path in: query - schema: *ref_72 + schema: *ref_85 - name: script_hash description: mask to filter exact matching path in: query - schema: *ref_73 + schema: *ref_86 - name: started_before description: filter on started before (inclusive) timestamp in: query - schema: *ref_74 + schema: *ref_87 - name: started_after description: filter on started after (exclusive) timestamp in: query - schema: *ref_75 + schema: *ref_88 - name: created_before description: filter on created before (inclusive) timestamp in: query - schema: &ref_128 - type: string - format: date-time + schema: *ref_101 - name: created_after description: filter on created after (exclusive) timestamp in: query - schema: &ref_129 - type: string - format: date-time + schema: *ref_102 - name: created_or_started_before description: >- filter on created_at for non non started job and started_at otherwise before (inclusive) timestamp in: query - schema: &ref_122 - type: string - format: date-time + schema: *ref_103 - name: running description: filter on running jobs in: query - schema: *ref_80 + schema: *ref_89 - name: scheduled_for_before_now description: filter on jobs scheduled_for before now (hence waitinf for a worker) in: query - schema: *ref_77 + schema: *ref_90 - name: created_or_started_after description: >- filter on created_at for non non started job and started_at otherwise after (exclusive) timestamp in: query - schema: &ref_123 - type: string - format: date-time + schema: *ref_104 - name: created_or_started_after_completed_jobs description: >- filter on created_at for non non started job and started_at otherwise after (exclusive) timestamp but only for the completed jobs in: query - schema: &ref_124 - type: string - format: date-time + schema: *ref_105 - name: job_kinds description: >- filter on job kind (values 'preview', 'script', 'dependencies', 'flow') separated by, in: query - schema: *ref_78 + schema: *ref_91 - name: suspended description: filter on suspended jobs in: query - schema: *ref_79 + schema: *ref_92 - name: args description: >- filter on jobs containing those args as a json subset (@> in postgres) in: query - schema: *ref_81 + schema: *ref_93 - name: tag description: filter on jobs with a given tag/worker group in: query - schema: *ref_83 + schema: *ref_94 - name: result description: >- filter on jobs containing those result as a json subset (@> in postgres) in: query - schema: *ref_82 + schema: *ref_95 + - name: allow_wildcards + description: allow wildcards (*) in the filter of label, tag, worker + in: query + schema: *ref_97 - name: page description: which page to return (start at 1, default 1) in: query @@ -9517,11 +11054,11 @@ paths: schema: type: array items: - oneOf: &ref_92 + oneOf: &ref_111 - allOf: - type: object - properties: *ref_87 - required: *ref_88 + properties: *ref_106 + required: *ref_107 - type: object properties: type: @@ -9530,15 +11067,15 @@ paths: - CompletedJob - allOf: - type: object - properties: *ref_89 - required: *ref_90 + properties: *ref_108 + required: *ref_109 - type: object properties: type: type: string enum: - QueuedJob - discriminator: &ref_93 + discriminator: &ref_112 propertyName: type /jobs/db_clock: get: @@ -9605,7 +11142,7 @@ paths: - name: id in: path required: true - schema: *ref_91 + schema: *ref_110 - name: no_logs in: query schema: @@ -9616,8 +11153,8 @@ paths: content: application/json: schema: - oneOf: *ref_92 - discriminator: *ref_93 + oneOf: *ref_111 + discriminator: *ref_112 /w/{workspace}/jobs_u/get_root_job_id/{id}: get: summary: get root job id @@ -9632,7 +11169,7 @@ paths: - name: id in: path required: true - schema: *ref_91 + schema: *ref_110 responses: '200': description: get root job id @@ -9654,7 +11191,7 @@ paths: - name: id in: path required: true - schema: *ref_91 + schema: *ref_110 responses: '200': description: job details @@ -9676,7 +11213,7 @@ paths: - name: id in: path required: true - schema: *ref_91 + schema: *ref_110 responses: '200': description: job args @@ -9697,7 +11234,7 @@ paths: - name: id in: path required: true - schema: *ref_91 + schema: *ref_110 - name: running in: query schema: @@ -9732,9 +11269,9 @@ paths: type: integer flow_status: type: object - additionalProperties: &ref_160 + additionalProperties: &ref_225 type: object - properties: &ref_161 + properties: &ref_226 scheduled_for: type: string format: date-time @@ -9766,7 +11303,8 @@ paths: description: job log content: text/plain: - type: string + schema: + type: string /w/{workspace}/jobs_u/get_flow_debug_info/{id}: get: summary: get flow debug info @@ -9781,7 +11319,7 @@ paths: - name: id in: path required: true - schema: *ref_91 + schema: *ref_110 responses: '200': description: flow debug info details @@ -9802,7 +11340,7 @@ paths: - name: id in: path required: true - schema: *ref_91 + schema: *ref_110 responses: '200': description: job details @@ -9810,8 +11348,8 @@ paths: application/json: schema: type: object - properties: *ref_87 - required: *ref_88 + properties: *ref_106 + required: *ref_107 /w/{workspace}/jobs_u/completed/get_result/{id}: get: summary: get completed job result @@ -9826,7 +11364,7 @@ paths: - name: id in: path required: true - schema: *ref_91 + schema: *ref_110 - name: suspended_job in: query schema: @@ -9863,10 +11401,10 @@ paths: - name: id in: path required: true - schema: *ref_91 + schema: *ref_110 - name: get_started in: query - schema: &ref_135 + schema: &ref_194 type: boolean responses: '200': @@ -9900,7 +11438,7 @@ paths: - name: id in: path required: true - schema: *ref_91 + schema: *ref_110 responses: '200': description: job details @@ -9908,8 +11446,8 @@ paths: application/json: schema: type: object - properties: *ref_87 - required: *ref_88 + properties: *ref_106 + required: *ref_107 /w/{workspace}/jobs_u/queue/cancel/{id}: post: summary: cancel queued or running job @@ -9924,7 +11462,7 @@ paths: - name: id in: path required: true - schema: *ref_91 + schema: *ref_110 requestBody: description: reason required: true @@ -9956,7 +11494,7 @@ paths: - name: path in: path required: true - schema: *ref_23 + schema: *ref_26 requestBody: description: reason required: true @@ -9988,7 +11526,7 @@ paths: - name: id in: path required: true - schema: *ref_91 + schema: *ref_110 requestBody: description: reason required: true @@ -10020,7 +11558,7 @@ paths: - name: id in: path required: true - schema: *ref_91 + schema: *ref_110 - name: resume_id in: path required: true @@ -10051,7 +11589,7 @@ paths: - name: id in: path required: true - schema: *ref_91 + schema: *ref_110 - name: resume_id in: path required: true @@ -10079,6 +11617,108 @@ paths: - approvalPage - resume - cancel + /w/{workspace}/jobs/slack_approval/{id}: + get: + summary: generate interactive slack approval for suspended job + operationId: getSlackApprovalPayload + tags: + - job + parameters: + - name: workspace + in: path + required: true + schema: *ref_0 + - name: id + in: path + required: true + schema: *ref_110 + - name: approver + in: query + schema: + type: string + - name: message + in: query + schema: + type: string + - name: slack_resource_path + in: query + required: true + schema: + type: string + - name: channel_id + in: query + required: true + schema: + type: string + - name: flow_step_id + in: query + required: true + schema: + type: string + - name: default_args_json + in: query + required: false + schema: + type: string + - name: dynamic_enums_json + in: query + required: false + schema: + type: string + responses: + '200': + description: Interactive slack approval message sent successfully + /w/{workspace}/jobs/teams_approval/{id}: + get: + summary: generate interactive teams approval for suspended job + operationId: getTeamsApprovalPayload + tags: + - job + parameters: + - name: workspace + in: path + required: true + schema: *ref_0 + - name: id + in: path + required: true + schema: *ref_110 + - name: approver + in: query + schema: + type: string + - name: message + in: query + schema: + type: string + - name: team_name + in: query + required: true + schema: + type: string + - name: channel_name + in: query + required: true + schema: + type: string + - name: flow_step_id + in: query + required: true + schema: + type: string + - name: default_args_json + in: query + required: false + schema: + type: string + - name: dynamic_enums_json + in: query + required: false + schema: + type: string + responses: + '200': + description: Interactive slack approval message sent successfully /w/{workspace}/jobs_u/resume/{id}/{resume_id}/{signature}: get: summary: resume a job for a suspended flow @@ -10093,7 +11733,7 @@ paths: - name: id in: path required: true - schema: *ref_91 + schema: *ref_110 - name: payload description: > The base64 encoded payload that has been encoded as a JSON. e.g how @@ -10101,7 +11741,7 @@ paths: `encodeURIComponent(btoa(JSON.stringify({a: 2})))` in: query - schema: *ref_94 + schema: *ref_113 - name: resume_id in: path required: true @@ -10136,7 +11776,7 @@ paths: - name: id in: path required: true - schema: *ref_91 + schema: *ref_110 - name: resume_id in: path required: true @@ -10178,7 +11818,7 @@ paths: - name: id in: path required: true - schema: *ref_91 + schema: *ref_110 - name: key in: path required: true @@ -10210,7 +11850,7 @@ paths: - name: id in: path required: true - schema: *ref_91 + schema: *ref_110 - name: key in: path required: true @@ -10236,7 +11876,7 @@ paths: - name: id in: path required: true - schema: *ref_91 + schema: *ref_110 requestBody: required: true content: @@ -10264,7 +11904,7 @@ paths: - name: id in: path required: true - schema: *ref_91 + schema: *ref_110 - name: resume_id in: path required: true @@ -10299,7 +11939,7 @@ paths: - name: id in: path required: true - schema: *ref_91 + schema: *ref_110 - name: resume_id in: path required: true @@ -10341,7 +11981,7 @@ paths: - name: id in: path required: true - schema: *ref_91 + schema: *ref_110 - name: resume_id in: path required: true @@ -10365,8 +12005,8 @@ paths: type: object properties: job: - oneOf: *ref_92 - discriminator: *ref_93 + oneOf: *ref_111 + discriminator: *ref_112 approvers: type: array items: @@ -10400,6 +12040,8 @@ paths: type: string timezone: type: string + cron_version: + type: string required: - schedule - timezone @@ -10431,7 +12073,7 @@ paths: application/json: schema: type: object - properties: &ref_171 + properties: &ref_236 path: type: string schedule: @@ -10444,7 +12086,7 @@ paths: type: boolean args: type: object - additionalProperties: *ref_18 + additionalProperties: *ref_21 enabled: type: boolean on_failure: @@ -10455,34 +12097,38 @@ paths: type: boolean on_failure_extra_args: type: object - additionalProperties: *ref_18 + additionalProperties: *ref_21 on_recovery: type: string on_recovery_times: type: number on_recovery_extra_args: type: object - additionalProperties: *ref_18 + additionalProperties: *ref_21 on_success: type: string on_success_extra_args: type: object - additionalProperties: *ref_18 + additionalProperties: *ref_21 ws_error_handler_muted: type: boolean retry: type: object - properties: *ref_95 + properties: *ref_114 no_flow_overlap: type: boolean summary: type: string + description: + type: string tag: type: string paused_until: type: string format: date-time - required: &ref_172 + cron_version: + type: string + required: &ref_237 - path - schedule - timezone @@ -10510,7 +12156,7 @@ paths: - name: path in: path required: true - schema: *ref_23 + schema: *ref_26 requestBody: description: updated schedule required: true @@ -10518,14 +12164,14 @@ paths: application/json: schema: type: object - properties: &ref_173 + properties: &ref_238 schedule: type: string timezone: type: string args: type: object - additionalProperties: *ref_18 + additionalProperties: *ref_21 on_failure: type: string on_failure_times: @@ -10534,34 +12180,38 @@ paths: type: boolean on_failure_extra_args: type: object - additionalProperties: *ref_18 + additionalProperties: *ref_21 on_recovery: type: string on_recovery_times: type: number on_recovery_extra_args: type: object - additionalProperties: *ref_18 + additionalProperties: *ref_21 on_success: type: string on_success_extra_args: type: object - additionalProperties: *ref_18 + additionalProperties: *ref_21 ws_error_handler_muted: type: boolean retry: type: object - properties: *ref_95 + properties: *ref_114 no_flow_overlap: type: boolean summary: type: string + description: + type: string tag: type: string paused_until: type: string format: date-time - required: &ref_174 + cron_version: + type: string + required: &ref_239 - schedule - timezone - script_path @@ -10588,7 +12238,7 @@ paths: - name: path in: path required: true - schema: *ref_23 + schema: *ref_26 requestBody: description: updated schedule enable required: true @@ -10622,7 +12272,7 @@ paths: - name: path in: path required: true - schema: *ref_23 + schema: *ref_26 responses: '200': description: schedule deleted @@ -10644,7 +12294,7 @@ paths: - name: path in: path required: true - schema: *ref_23 + schema: *ref_26 responses: '200': description: schedule deleted @@ -10652,7 +12302,7 @@ paths: application/json: schema: type: object - properties: &ref_96 + properties: &ref_115 path: type: string edited_by: @@ -10672,7 +12322,7 @@ paths: type: boolean args: type: object - additionalProperties: *ref_18 + additionalProperties: *ref_21 extra_perms: type: object additionalProperties: @@ -10689,26 +12339,28 @@ paths: type: boolean on_failure_extra_args: type: object - additionalProperties: *ref_18 + additionalProperties: *ref_21 on_recovery: type: string on_recovery_times: type: number on_recovery_extra_args: type: object - additionalProperties: *ref_18 + additionalProperties: *ref_21 on_success: type: string on_success_extra_args: type: object - additionalProperties: *ref_18 + additionalProperties: *ref_21 ws_error_handler_muted: type: boolean retry: type: object - properties: *ref_95 + properties: *ref_114 summary: type: string + description: + type: string no_flow_overlap: type: boolean tag: @@ -10716,7 +12368,9 @@ paths: paused_until: type: string format: date-time - required: &ref_97 + cron_version: + type: string + required: &ref_116 - path - edited_by - edited_at @@ -10741,7 +12395,7 @@ paths: - name: path in: path required: true - schema: *ref_23 + schema: *ref_26 responses: '200': description: schedule exists @@ -10773,7 +12427,7 @@ paths: filter on jobs containing those args as a json subset (@> in postgres) in: query - schema: *ref_81 + schema: *ref_93 - name: path description: filter by path in: query @@ -10796,8 +12450,8 @@ paths: type: array items: type: object - properties: *ref_96 - required: *ref_97 + properties: *ref_115 + required: *ref_116 /w/{workspace}/schedules/list_with_jobs: get: summary: list schedules with last 20 jobs @@ -10825,10 +12479,10 @@ paths: schema: type: array items: - allOf: &ref_170 + allOf: &ref_235 - type: object - properties: *ref_96 - required: *ref_97 + properties: *ref_115 + required: *ref_116 - type: object properties: jobs: @@ -10907,13 +12561,15 @@ paths: application/json: schema: type: object - properties: &ref_175 + properties: &ref_240 path: type: string script_path: type: string route_path: type: string + workspaced_route: + type: boolean static_asset_config: type: object properties: @@ -10935,18 +12591,34 @@ paths: - put - delete - patch + authentication_resource_path: + type: string is_async: type: boolean - requires_auth: + authentication_method: + type: string + enum: &ref_117 + - none + - windmill + - api_key + - basic_http + - custom_script + - signature + is_static_website: type: boolean - required: &ref_176 + wrap_body: + type: boolean + raw_string: + type: boolean + required: &ref_241 - path - script_path - route_path - is_flow - is_async - - requires_auth + - authentication_method - http_method + - is_static_website responses: '201': description: http trigger created @@ -10968,7 +12640,7 @@ paths: - name: path in: path required: true - schema: *ref_23 + schema: *ref_26 requestBody: description: updated trigger required: true @@ -10976,13 +12648,15 @@ paths: application/json: schema: type: object - properties: &ref_177 + properties: &ref_242 path: type: string script_path: type: string route_path: type: string + workspaced_route: + type: boolean static_asset_config: type: object properties: @@ -10994,6 +12668,8 @@ paths: type: string required: - s3 + authentication_resource_path: + type: string is_flow: type: boolean http_method: @@ -11006,16 +12682,24 @@ paths: - patch is_async: type: boolean - requires_auth: + authentication_method: + type: string + enum: *ref_117 + is_static_website: type: boolean - required: &ref_178 + wrap_body: + type: boolean + raw_string: + type: boolean + required: &ref_243 - path - script_path - is_flow - kind - is_async - - requires_auth + - authentication_method - http_method + - is_static_website responses: '200': description: http trigger updated @@ -11037,7 +12721,7 @@ paths: - name: path in: path required: true - schema: *ref_23 + schema: *ref_26 responses: '200': description: http trigger deleted @@ -11059,24 +12743,46 @@ paths: - name: path in: path required: true - schema: *ref_23 + schema: *ref_26 responses: '200': description: http trigger deleted content: application/json: schema: + allOf: &ref_118 + - type: object + properties: &ref_122 + path: + type: string + script_path: + type: string + email: + type: string + extra_perms: + type: object + additionalProperties: + type: boolean + workspace_id: + type: string + edited_by: + type: string + edited_at: + type: string + format: date-time + is_flow: + type: boolean + required: &ref_123 + - path + - script_path + - email + - extra_perms + - workspace_id + - edited_by + - edited_at + - is_flow type: object - properties: &ref_98 - path: - type: string - edited_by: - type: string - edited_at: - type: string - format: date-time - script_path: - type: string + properties: &ref_119 route_path: type: string static_asset_config: @@ -11090,16 +12796,6 @@ paths: type: string required: - s3 - is_flow: - type: boolean - extra_perms: - type: object - additionalProperties: - type: boolean - email: - type: string - workspace_id: - type: string http_method: type: string enum: @@ -11108,23 +12804,30 @@ paths: - put - delete - patch + authentication_resource_path: + type: string is_async: type: boolean - requires_auth: + authentication_method: + type: string + enum: *ref_117 + is_static_website: type: boolean - required: &ref_99 - - path - - edited_by - - edited_at - - script_path + workspaced_route: + type: boolean + wrap_body: + type: boolean + raw_string: + type: boolean + required: &ref_120 - route_path - - extra_perms - - is_flow - - email - - workspace_id - is_async - - requires_auth + - authentication_method - http_method + - is_static_website + - workspaced_route + - wrap_body + - raw_string /w/{workspace}/http_triggers/list: get: summary: list http triggers @@ -11165,9 +12868,10 @@ paths: schema: type: array items: + allOf: *ref_118 type: object - properties: *ref_98 - required: *ref_99 + properties: *ref_119 + required: *ref_120 /w/{workspace}/http_triggers/exists/{path}: get: summary: does http trigger exists @@ -11182,7 +12886,7 @@ paths: - name: path in: path required: true - schema: *ref_23 + schema: *ref_26 responses: '200': description: http trigger exists @@ -11219,8 +12923,11 @@ paths: - put - delete - patch + trigger_path: + type: string + workspaced_route: + type: boolean required: - - kind - route_path - http_method responses: @@ -11248,7 +12955,7 @@ paths: application/json: schema: type: object - properties: &ref_179 + properties: &ref_244 path: type: string script_path: @@ -11273,7 +12980,7 @@ paths: initial_messages: type: array items: - anyOf: &ref_100 + anyOf: &ref_121 - type: object properties: raw_message: @@ -11289,7 +12996,7 @@ paths: type: string args: type: object - additionalProperties: *ref_18 + additionalProperties: *ref_21 is_flow: type: boolean required: @@ -11300,15 +13007,16 @@ paths: - runnable_result url_runnable_args: type: object - additionalProperties: *ref_18 - required: &ref_180 + additionalProperties: *ref_21 + can_return_message: + type: boolean + required: &ref_245 - path - script_path - url - is_flow - filters - - initial_messages - - url_runnable_args + - can_return_message responses: '201': description: websocket trigger created @@ -11330,7 +13038,7 @@ paths: - name: path in: path required: true - schema: *ref_23 + schema: *ref_26 requestBody: description: updated trigger required: true @@ -11338,7 +13046,7 @@ paths: application/json: schema: type: object - properties: &ref_181 + properties: &ref_246 url: type: string path: @@ -11361,18 +13069,19 @@ paths: initial_messages: type: array items: - anyOf: *ref_100 + anyOf: *ref_121 url_runnable_args: type: object - additionalProperties: *ref_18 - required: &ref_182 + additionalProperties: *ref_21 + can_return_message: + type: boolean + required: &ref_247 - path - script_path - url - is_flow - filters - - initial_messages - - url_runnable_args + - can_return_message responses: '200': description: websocket trigger updated @@ -11394,7 +13103,7 @@ paths: - name: path in: path required: true - schema: *ref_23 + schema: *ref_26 responses: '200': description: websocket trigger deleted @@ -11416,36 +13125,21 @@ paths: - name: path in: path required: true - schema: *ref_23 + schema: *ref_26 responses: '200': description: websocket trigger deleted content: application/json: schema: + allOf: &ref_124 + - type: object + properties: *ref_122 + required: *ref_123 type: object - properties: &ref_101 - path: - type: string - edited_by: - type: string - edited_at: - type: string - format: date-time - script_path: - type: string + properties: &ref_125 url: type: string - is_flow: - type: boolean - extra_perms: - type: object - additionalProperties: - type: boolean - email: - type: string - workspace_id: - type: string server_id: type: string last_server_ping: @@ -11469,24 +13163,17 @@ paths: initial_messages: type: array items: - anyOf: *ref_100 + anyOf: *ref_121 url_runnable_args: type: object - additionalProperties: *ref_18 - required: &ref_102 - - path - - edited_by - - edited_at - - script_path + additionalProperties: *ref_21 + can_return_message: + type: boolean + required: &ref_126 - url - - extra_perms - - is_flow - - email - - workspace_id - enabled - filters - - initial_messages - - url_runnable_args + - can_return_message /w/{workspace}/websocket_triggers/list: get: summary: list websocket triggers @@ -11527,9 +13214,10 @@ paths: schema: type: array items: + allOf: *ref_124 type: object - properties: *ref_101 - required: *ref_102 + properties: *ref_125 + required: *ref_126 /w/{workspace}/websocket_triggers/exists/{path}: get: summary: does websocket trigger exists @@ -11544,7 +13232,7 @@ paths: - name: path in: path required: true - schema: *ref_23 + schema: *ref_26 responses: '200': description: websocket trigger exists @@ -11566,7 +13254,7 @@ paths: - name: path in: path required: true - schema: *ref_23 + schema: *ref_26 requestBody: description: updated websocket trigger enable required: true @@ -11586,6 +13274,2400 @@ paths: text/plain: schema: type: string + /w/{workspace}/websocket_triggers/test: + post: + summary: test websocket connection + operationId: testWebsocketConnection + tags: + - websocket_trigger + parameters: + - name: workspace + in: path + required: true + schema: *ref_0 + requestBody: + description: test websocket connection + required: true + content: + application/json: + schema: + type: object + properties: + url: + type: string + url_runnable_args: + type: object + additionalProperties: *ref_21 + can_return_message: + type: boolean + required: + - url + - can_return_message + responses: + '200': + description: successfuly connected to websocket + content: + text/plain: + schema: + type: string + /w/{workspace}/kafka_triggers/create: + post: + summary: create kafka trigger + operationId: createKafkaTrigger + tags: + - kafka_trigger + parameters: + - name: workspace + in: path + required: true + schema: *ref_0 + requestBody: + description: new kafka trigger + required: true + content: + application/json: + schema: + type: object + properties: &ref_270 + path: + type: string + script_path: + type: string + is_flow: + type: boolean + kafka_resource_path: + type: string + group_id: + type: string + topics: + type: array + items: + type: string + enabled: + type: boolean + required: &ref_271 + - path + - script_path + - is_flow + - kafka_resource_path + - group_id + - topics + responses: + '201': + description: kafka trigger created + content: + text/plain: + schema: + type: string + /w/{workspace}/kafka_triggers/update/{path}: + post: + summary: update kafka trigger + operationId: updateKafkaTrigger + tags: + - kafka_trigger + parameters: + - name: workspace + in: path + required: true + schema: *ref_0 + - name: path + in: path + required: true + schema: *ref_26 + requestBody: + description: updated trigger + required: true + content: + application/json: + schema: + type: object + properties: &ref_272 + kafka_resource_path: + type: string + group_id: + type: string + topics: + type: array + items: + type: string + path: + type: string + script_path: + type: string + is_flow: + type: boolean + required: &ref_273 + - path + - script_path + - kafka_resource_path + - group_id + - topics + - is_flow + responses: + '200': + description: kafka trigger updated + content: + text/plain: + schema: + type: string + /w/{workspace}/kafka_triggers/delete/{path}: + delete: + summary: delete kafka trigger + operationId: deleteKafkaTrigger + tags: + - kafka_trigger + parameters: + - name: workspace + in: path + required: true + schema: *ref_0 + - name: path + in: path + required: true + schema: *ref_26 + responses: + '200': + description: kafka trigger deleted + content: + text/plain: + schema: + type: string + /w/{workspace}/kafka_triggers/get/{path}: + get: + summary: get kafka trigger + operationId: getKafkaTrigger + tags: + - kafka_trigger + parameters: + - name: workspace + in: path + required: true + schema: *ref_0 + - name: path + in: path + required: true + schema: *ref_26 + responses: + '200': + description: kafka trigger deleted + content: + application/json: + schema: + allOf: &ref_127 + - type: object + properties: *ref_122 + required: *ref_123 + type: object + properties: &ref_128 + kafka_resource_path: + type: string + group_id: + type: string + topics: + type: array + items: + type: string + server_id: + type: string + last_server_ping: + type: string + format: date-time + error: + type: string + enabled: + type: boolean + required: &ref_129 + - kafka_resource_path + - group_id + - topics + - enabled + /w/{workspace}/kafka_triggers/list: + get: + summary: list kafka triggers + operationId: listKafkaTriggers + tags: + - kafka_trigger + parameters: + - required: true + name: workspace + in: path + schema: *ref_0 + - name: page + description: which page to return (start at 1, default 1) + in: query + schema: *ref_5 + - name: per_page + description: number of items to return for a given page (default 30, max 100) + in: query + schema: *ref_6 + - name: path + description: filter by path + in: query + schema: + type: string + - name: is_flow + in: query + schema: + type: boolean + - name: path_start + in: query + schema: + type: string + responses: + '200': + description: kafka trigger list + content: + application/json: + schema: + type: array + items: + allOf: *ref_127 + type: object + properties: *ref_128 + required: *ref_129 + /w/{workspace}/kafka_triggers/exists/{path}: + get: + summary: does kafka trigger exists + operationId: existsKafkaTrigger + tags: + - kafka_trigger + parameters: + - name: workspace + in: path + required: true + schema: *ref_0 + - name: path + in: path + required: true + schema: *ref_26 + responses: + '200': + description: kafka trigger exists + content: + application/json: + schema: + type: boolean + /w/{workspace}/kafka_triggers/setenabled/{path}: + post: + summary: set enabled kafka trigger + operationId: setKafkaTriggerEnabled + tags: + - kafka_trigger + parameters: + - name: workspace + in: path + required: true + schema: *ref_0 + - name: path + in: path + required: true + schema: *ref_26 + requestBody: + description: updated kafka trigger enable + required: true + content: + application/json: + schema: + type: object + properties: + enabled: + type: boolean + required: + - enabled + responses: + '200': + description: kafka trigger enabled set + content: + text/plain: + schema: + type: string + /w/{workspace}/kafka_triggers/test: + post: + summary: test kafka connection + operationId: testKafkaConnection + tags: + - kafka_trigger + parameters: + - name: workspace + in: path + required: true + schema: *ref_0 + requestBody: + description: test kafka connection + required: true + content: + application/json: + schema: + type: object + properties: + connection: + type: object + required: + - connection + responses: + '200': + description: successfuly connected to kafka brokers + content: + text/plain: + schema: + type: string + /w/{workspace}/nats_triggers/create: + post: + summary: create nats trigger + operationId: createNatsTrigger + tags: + - nats_trigger + parameters: + - name: workspace + in: path + required: true + schema: *ref_0 + requestBody: + description: new nats trigger + required: true + content: + application/json: + schema: + type: object + properties: &ref_274 + path: + type: string + script_path: + type: string + is_flow: + type: boolean + nats_resource_path: + type: string + use_jetstream: + type: boolean + stream_name: + type: string + consumer_name: + type: string + subjects: + type: array + items: + type: string + enabled: + type: boolean + required: &ref_275 + - path + - script_path + - is_flow + - nats_resource_path + - use_jetstream + - subjects + responses: + '201': + description: nats trigger created + content: + text/plain: + schema: + type: string + /w/{workspace}/nats_triggers/update/{path}: + post: + summary: update nats trigger + operationId: updateNatsTrigger + tags: + - nats_trigger + parameters: + - name: workspace + in: path + required: true + schema: *ref_0 + - name: path + in: path + required: true + schema: *ref_26 + requestBody: + description: updated trigger + required: true + content: + application/json: + schema: + type: object + properties: &ref_276 + nats_resource_path: + type: string + use_jetstream: + type: boolean + stream_name: + type: string + consumer_name: + type: string + subjects: + type: array + items: + type: string + path: + type: string + script_path: + type: string + is_flow: + type: boolean + required: &ref_277 + - path + - script_path + - nats_resource_path + - use_jetstream + - subjects + - is_flow + responses: + '200': + description: nats trigger updated + content: + text/plain: + schema: + type: string + /w/{workspace}/nats_triggers/delete/{path}: + delete: + summary: delete nats trigger + operationId: deleteNatsTrigger + tags: + - nats_trigger + parameters: + - name: workspace + in: path + required: true + schema: *ref_0 + - name: path + in: path + required: true + schema: *ref_26 + responses: + '200': + description: nats trigger deleted + content: + text/plain: + schema: + type: string + /w/{workspace}/nats_triggers/get/{path}: + get: + summary: get nats trigger + operationId: getNatsTrigger + tags: + - nats_trigger + parameters: + - name: workspace + in: path + required: true + schema: *ref_0 + - name: path + in: path + required: true + schema: *ref_26 + responses: + '200': + description: nats trigger deleted + content: + application/json: + schema: + allOf: &ref_130 + - type: object + properties: *ref_122 + required: *ref_123 + type: object + properties: &ref_131 + nats_resource_path: + type: string + use_jetstream: + type: boolean + stream_name: + type: string + consumer_name: + type: string + subjects: + type: array + items: + type: string + server_id: + type: string + last_server_ping: + type: string + format: date-time + error: + type: string + enabled: + type: boolean + required: &ref_132 + - nats_resource_path + - use_jetstream + - subjects + - enabled + /w/{workspace}/nats_triggers/list: + get: + summary: list nats triggers + operationId: listNatsTriggers + tags: + - nats_trigger + parameters: + - required: true + name: workspace + in: path + schema: *ref_0 + - name: page + description: which page to return (start at 1, default 1) + in: query + schema: *ref_5 + - name: per_page + description: number of items to return for a given page (default 30, max 100) + in: query + schema: *ref_6 + - name: path + description: filter by path + in: query + schema: + type: string + - name: is_flow + in: query + schema: + type: boolean + - name: path_start + in: query + schema: + type: string + responses: + '200': + description: nats trigger list + content: + application/json: + schema: + type: array + items: + allOf: *ref_130 + type: object + properties: *ref_131 + required: *ref_132 + /w/{workspace}/nats_triggers/exists/{path}: + get: + summary: does nats trigger exists + operationId: existsNatsTrigger + tags: + - nats_trigger + parameters: + - name: workspace + in: path + required: true + schema: *ref_0 + - name: path + in: path + required: true + schema: *ref_26 + responses: + '200': + description: nats trigger exists + content: + application/json: + schema: + type: boolean + /w/{workspace}/nats_triggers/setenabled/{path}: + post: + summary: set enabled nats trigger + operationId: setNatsTriggerEnabled + tags: + - nats_trigger + parameters: + - name: workspace + in: path + required: true + schema: *ref_0 + - name: path + in: path + required: true + schema: *ref_26 + requestBody: + description: updated nats trigger enable + required: true + content: + application/json: + schema: + type: object + properties: + enabled: + type: boolean + required: + - enabled + responses: + '200': + description: nats trigger enabled set + content: + text/plain: + schema: + type: string + /w/{workspace}/nats_triggers/test: + post: + summary: test NATS connection + operationId: testNatsConnection + tags: + - nats_trigger + parameters: + - name: workspace + in: path + required: true + schema: *ref_0 + requestBody: + description: test nats connection + required: true + content: + application/json: + schema: + type: object + properties: + connection: + type: object + required: + - connection + responses: + '200': + description: successfuly connected to NATS servers + content: + text/plain: + schema: + type: string + /w/{workspace}/sqs_triggers/create: + post: + summary: create sqs trigger + operationId: createSqsTrigger + tags: + - sqs_trigger + parameters: + - name: workspace + in: path + required: true + schema: *ref_0 + requestBody: + description: new sqs trigger + required: true + content: + application/json: + schema: + type: object + properties: &ref_257 + queue_url: + type: string + aws_auth_resource_type: + type: string + enum: &ref_133 + - oidc + - credentials + aws_resource_path: + type: string + message_attributes: + type: array + items: + type: string + path: + type: string + script_path: + type: string + is_flow: + type: boolean + enabled: + type: boolean + required: &ref_258 + - queue_url + - aws_resource_path + - path + - script_path + - is_flow + - aws_auth_resource_type + responses: + '201': + description: sqs trigger created + content: + text/plain: + schema: + type: string + /w/{workspace}/sqs_triggers/update/{path}: + post: + summary: update sqs trigger + operationId: updateSqsTrigger + tags: + - sqs_trigger + parameters: + - name: workspace + in: path + required: true + schema: *ref_0 + - name: path + in: path + required: true + schema: *ref_26 + requestBody: + description: updated trigger + required: true + content: + application/json: + schema: + type: object + properties: &ref_259 + queue_url: + type: string + aws_auth_resource_type: + type: string + enum: *ref_133 + aws_resource_path: + type: string + message_attributes: + type: array + items: + type: string + path: + type: string + script_path: + type: string + is_flow: + type: boolean + enabled: + type: boolean + required: &ref_260 + - queue_url + - aws_resource_path + - path + - script_path + - is_flow + - enabled + - aws_auth_resource_type + responses: + '200': + description: sqs trigger updated + content: + text/plain: + schema: + type: string + /w/{workspace}/sqs_triggers/delete/{path}: + delete: + summary: delete sqs trigger + operationId: deleteSqsTrigger + tags: + - sqs_trigger + parameters: + - name: workspace + in: path + required: true + schema: *ref_0 + - name: path + in: path + required: true + schema: *ref_26 + responses: + '200': + description: sqs trigger deleted + content: + text/plain: + schema: + type: string + /w/{workspace}/sqs_triggers/get/{path}: + get: + summary: get sqs trigger + operationId: getSqsTrigger + tags: + - sqs_trigger + parameters: + - name: workspace + in: path + required: true + schema: *ref_0 + - name: path + in: path + required: true + schema: *ref_26 + responses: + '200': + description: sqs trigger deleted + content: + application/json: + schema: + allOf: &ref_134 + - type: object + properties: *ref_122 + required: *ref_123 + type: object + properties: &ref_135 + queue_url: + type: string + aws_auth_resource_type: + type: string + enum: *ref_133 + aws_resource_path: + type: string + message_attributes: + type: array + items: + type: string + server_id: + type: string + last_server_ping: + type: string + format: date-time + error: + type: string + enabled: + type: boolean + required: &ref_136 + - queue_url + - aws_resource_path + - enabled + - aws_auth_resource_type + /w/{workspace}/sqs_triggers/list: + get: + summary: list sqs triggers + operationId: listSqsTriggers + tags: + - sqs_trigger + parameters: + - required: true + name: workspace + in: path + schema: *ref_0 + - name: page + description: which page to return (start at 1, default 1) + in: query + schema: *ref_5 + - name: per_page + description: number of items to return for a given page (default 30, max 100) + in: query + schema: *ref_6 + - name: path + description: filter by path + in: query + schema: + type: string + - name: is_flow + in: query + schema: + type: boolean + - name: path_start + in: query + schema: + type: string + responses: + '200': + description: sqs trigger list + content: + application/json: + schema: + type: array + items: + allOf: *ref_134 + type: object + properties: *ref_135 + required: *ref_136 + /w/{workspace}/sqs_triggers/exists/{path}: + get: + summary: does sqs trigger exists + operationId: existsSqsTrigger + tags: + - sqs_trigger + parameters: + - name: workspace + in: path + required: true + schema: *ref_0 + - name: path + in: path + required: true + schema: *ref_26 + responses: + '200': + description: sqs trigger exists + content: + application/json: + schema: + type: boolean + /w/{workspace}/sqs_triggers/setenabled/{path}: + post: + summary: set enabled sqs trigger + operationId: setSqsTriggerEnabled + tags: + - sqs_trigger + parameters: + - name: workspace + in: path + required: true + schema: *ref_0 + - name: path + in: path + required: true + schema: *ref_26 + requestBody: + description: updated sqs trigger enable + required: true + content: + application/json: + schema: + type: object + properties: + enabled: + type: boolean + required: + - enabled + responses: + '200': + description: sqs trigger enabled set + content: + text/plain: + schema: + type: string + /w/{workspace}/sqs_triggers/test: + post: + summary: test sqs connection + operationId: testSqsConnection + tags: + - sqs_trigger + parameters: + - name: workspace + in: path + required: true + schema: *ref_0 + requestBody: + description: test sqs connection + required: true + content: + application/json: + schema: + type: object + properties: + connection: + type: object + required: + - connection + responses: + '200': + description: successfuly connected to sqs + content: + text/plain: + schema: + type: string + /w/{workspace}/mqtt_triggers/create: + post: + summary: create mqtt trigger + operationId: createMqttTrigger + tags: + - mqtt_trigger + parameters: + - name: workspace + in: path + required: true + schema: *ref_0 + requestBody: + description: new mqtt trigger + required: true + content: + application/json: + schema: + type: object + properties: &ref_249 + mqtt_resource_path: + type: string + subscribe_topics: + type: array + items: + type: object + properties: &ref_137 + qos: + type: string + enum: &ref_248 + - qos0 + - qos1 + - qos2 + topic: + type: string + required: &ref_138 + - qos + - topic + client_id: + type: string + v3_config: + type: object + properties: &ref_139 + clean_session: + type: boolean + v5_config: + type: object + properties: &ref_140 + clean_start: + type: boolean + topic_alias: + type: number + session_expiry_interval: + type: number + client_version: + type: string + enum: &ref_141 + - v3 + - v5 + path: + type: string + script_path: + type: string + is_flow: + type: boolean + enabled: + type: boolean + required: &ref_250 + - path + - script_path + - is_flow + - subscribe_topics + - mqtt_resource_path + responses: + '201': + description: mqtt trigger created + content: + text/plain: + schema: + type: string + /w/{workspace}/mqtt_triggers/update/{path}: + post: + summary: update mqtt trigger + operationId: updateMqttTrigger + tags: + - mqtt_trigger + parameters: + - name: workspace + in: path + required: true + schema: *ref_0 + - name: path + in: path + required: true + schema: *ref_26 + requestBody: + description: updated trigger + required: true + content: + application/json: + schema: + type: object + properties: &ref_251 + mqtt_resource_path: + type: string + subscribe_topics: + type: array + items: + type: object + properties: *ref_137 + required: *ref_138 + client_id: + type: string + v3_config: + type: object + properties: *ref_139 + v5_config: + type: object + properties: *ref_140 + client_version: + type: string + enum: *ref_141 + path: + type: string + script_path: + type: string + is_flow: + type: boolean + enabled: + type: boolean + required: &ref_252 + - path + - script_path + - is_flow + - enabled + - subscribe_topics + - mqtt_resource_path + responses: + '200': + description: mqtt trigger updated + content: + text/plain: + schema: + type: string + /w/{workspace}/mqtt_triggers/delete/{path}: + delete: + summary: delete mqtt trigger + operationId: deleteMqttTrigger + tags: + - mqtt_trigger + parameters: + - name: workspace + in: path + required: true + schema: *ref_0 + - name: path + in: path + required: true + schema: *ref_26 + responses: + '200': + description: mqtt trigger deleted + content: + text/plain: + schema: + type: string + /w/{workspace}/mqtt_triggers/get/{path}: + get: + summary: get mqtt trigger + operationId: getMqttTrigger + tags: + - mqtt_trigger + parameters: + - name: workspace + in: path + required: true + schema: *ref_0 + - name: path + in: path + required: true + schema: *ref_26 + responses: + '200': + description: mqtt trigger deleted + content: + application/json: + schema: + allOf: &ref_142 + - type: object + properties: *ref_122 + required: *ref_123 + type: object + properties: &ref_143 + mqtt_resource_path: + type: string + subscribe_topics: + type: array + items: + type: object + properties: *ref_137 + required: *ref_138 + v3_config: + type: object + properties: *ref_139 + v5_config: + type: object + properties: *ref_140 + client_id: + type: string + client_version: + type: string + enum: *ref_141 + server_id: + type: string + last_server_ping: + type: string + format: date-time + error: + type: string + enabled: + type: boolean + required: &ref_144 + - enabled + - subscribe_topics + - mqtt_resource_path + /w/{workspace}/mqtt_triggers/list: + get: + summary: list mqtt triggers + operationId: listMqttTriggers + tags: + - mqtt_trigger + parameters: + - required: true + name: workspace + in: path + schema: *ref_0 + - name: page + description: which page to return (start at 1, default 1) + in: query + schema: *ref_5 + - name: per_page + description: number of items to return for a given page (default 30, max 100) + in: query + schema: *ref_6 + - name: path + description: filter by path + in: query + schema: + type: string + - name: is_flow + in: query + schema: + type: boolean + - name: path_start + in: query + schema: + type: string + responses: + '200': + description: mqtt trigger list + content: + application/json: + schema: + type: array + items: + allOf: *ref_142 + type: object + properties: *ref_143 + required: *ref_144 + /w/{workspace}/mqtt_triggers/exists/{path}: + get: + summary: does mqtt trigger exists + operationId: existsMqttTrigger + tags: + - mqtt_trigger + parameters: + - name: workspace + in: path + required: true + schema: *ref_0 + - name: path + in: path + required: true + schema: *ref_26 + responses: + '200': + description: mqtt trigger exists + content: + application/json: + schema: + type: boolean + /w/{workspace}/mqtt_triggers/setenabled/{path}: + post: + summary: set enabled mqtt trigger + operationId: setMqttTriggerEnabled + tags: + - mqtt_trigger + parameters: + - name: workspace + in: path + required: true + schema: *ref_0 + - name: path + in: path + required: true + schema: *ref_26 + requestBody: + description: updated mqtt trigger enable + required: true + content: + application/json: + schema: + type: object + properties: + enabled: + type: boolean + required: + - enabled + responses: + '200': + description: mqtt trigger enabled set + content: + text/plain: + schema: + type: string + /w/{workspace}/mqtt_triggers/test: + post: + summary: test mqtt connection + operationId: testMqttConnection + tags: + - mqtt_trigger + parameters: + - name: workspace + in: path + required: true + schema: *ref_0 + requestBody: + description: test mqtt connection + required: true + content: + application/json: + schema: + type: object + properties: + connection: + type: object + required: + - connection + responses: + '200': + description: successfully connected to mqtt + content: + text/plain: + schema: + type: string + /w/{workspace}/gcp_triggers/create: + post: + summary: create gcp trigger + operationId: createGcpTrigger + tags: + - gcp_trigger + parameters: + - name: workspace + in: path + required: true + schema: *ref_0 + requestBody: + description: new gcp trigger + required: true + content: + application/json: + schema: + type: object + properties: &ref_145 + gcp_resource_path: + type: string + subscription_mode: + type: string + enum: &ref_150 + - existing + - create_update + description: >- + The mode of subscription. 'existing' means using an existing + GCP subscription, while 'create_update' involves creating or + updating a new subscription. + topic_id: + type: string + subscription_id: + type: string + base_endpoint: + type: string + delivery_type: + type: string + enum: &ref_147 + - push + - pull + delivery_config: + type: object + properties: &ref_148 + audience: + type: string + authenticate: + type: boolean + required: &ref_149 + - authenticate + - base_endpoint + path: + type: string + script_path: + type: string + is_flow: + type: boolean + enabled: + type: boolean + required: &ref_146 + - path + - script_path + - is_flow + - gcp_resource_path + - topic_id + - subscription_mode + responses: + '201': + description: gcp trigger created + content: + text/plain: + schema: + type: string + /w/{workspace}/gcp_triggers/update/{path}: + post: + summary: update gcp trigger + operationId: updateGcpTrigger + tags: + - gcp_trigger + parameters: + - name: workspace + in: path + required: true + schema: *ref_0 + - name: path + in: path + required: true + schema: *ref_26 + requestBody: + description: updated trigger + required: true + content: + application/json: + schema: + type: object + properties: *ref_145 + required: *ref_146 + responses: + '200': + description: gcp trigger updated + content: + text/plain: + schema: + type: string + /w/{workspace}/gcp_triggers/delete/{path}: + delete: + summary: delete gcp trigger + operationId: deleteGcpTrigger + tags: + - gcp_trigger + parameters: + - name: workspace + in: path + required: true + schema: *ref_0 + - name: path + in: path + required: true + schema: *ref_26 + responses: + '200': + description: gcp trigger deleted + content: + text/plain: + schema: + type: string + /w/{workspace}/gcp_triggers/get/{path}: + get: + summary: get gcp trigger + operationId: getGcpTrigger + tags: + - gcp_trigger + parameters: + - name: workspace + in: path + required: true + schema: *ref_0 + - name: path + in: path + required: true + schema: *ref_26 + responses: + '200': + description: gcp trigger deleted + content: + application/json: + schema: + allOf: &ref_151 + - type: object + properties: *ref_122 + required: *ref_123 + type: object + properties: &ref_152 + gcp_resource_path: + type: string + topic_id: + type: string + subscription_id: + type: string + server_id: + type: string + delivery_type: + type: string + enum: *ref_147 + delivery_config: + type: object + properties: *ref_148 + required: *ref_149 + subscription_mode: + type: string + enum: *ref_150 + description: >- + The mode of subscription. 'existing' means using an + existing GCP subscription, while 'create_update' involves + creating or updating a new subscription. + last_server_ping: + type: string + format: date-time + error: + type: string + enabled: + type: boolean + required: &ref_153 + - gcp_resource_path + - topic_id + - subscription_id + - enabled + - delivery_type + - subscription_mode + /w/{workspace}/gcp_triggers/list: + get: + summary: list gcp triggers + operationId: listGcpTriggers + tags: + - gcp_trigger + parameters: + - required: true + name: workspace + in: path + schema: *ref_0 + - name: page + description: which page to return (start at 1, default 1) + in: query + schema: *ref_5 + - name: per_page + description: number of items to return for a given page (default 30, max 100) + in: query + schema: *ref_6 + - name: path + description: filter by path + in: query + schema: + type: string + - name: is_flow + in: query + schema: + type: boolean + - name: path_start + in: query + schema: + type: string + responses: + '200': + description: gcp trigger list + content: + application/json: + schema: + type: array + items: + allOf: *ref_151 + type: object + properties: *ref_152 + required: *ref_153 + /w/{workspace}/gcp_triggers/exists/{path}: + get: + summary: does gcp trigger exists + operationId: existsGcpTrigger + tags: + - gcp_trigger + parameters: + - name: workspace + in: path + required: true + schema: *ref_0 + - name: path + in: path + required: true + schema: *ref_26 + responses: + '200': + description: gcp trigger exists + content: + application/json: + schema: + type: boolean + /w/{workspace}/gcp_triggers/setenabled/{path}: + post: + summary: set enabled gcp trigger + operationId: setGcpTriggerEnabled + tags: + - gcp_trigger + parameters: + - name: workspace + in: path + required: true + schema: *ref_0 + - name: path + in: path + required: true + schema: *ref_26 + requestBody: + description: updated gcp trigger enable + required: true + content: + application/json: + schema: + type: object + properties: + enabled: + type: boolean + required: + - enabled + responses: + '200': + description: gcp trigger enabled set + content: + text/plain: + schema: + type: string + /w/{workspace}/gcp_triggers/test: + post: + summary: test gcp connection + operationId: testGcpConnection + tags: + - gcp_trigger + parameters: + - name: workspace + in: path + required: true + schema: *ref_0 + requestBody: + description: test gcp connection + required: true + content: + application/json: + schema: + type: object + properties: + connection: + type: object + required: + - connection + responses: + '200': + description: try to connect to a gcp broker + content: + text/plain: + schema: + type: string + /w/{workspace}/gcp_triggers/subscriptions/delete/{path}: + delete: + summary: delete gcp trigger + operationId: deleteGcpSubscription + tags: + - gcp_trigger + parameters: + - name: workspace + in: path + required: true + schema: *ref_0 + - name: path + in: path + required: true + schema: *ref_26 + requestBody: + description: args to delete subscription from google cloud + required: true + content: + application/json: + schema: + type: object + properties: &ref_255 + subscription_id: + type: string + required: &ref_256 + - subscription_id + responses: + '200': + description: gcp trigger deleted + content: + text/plain: + schema: + type: string + /w/{workspace}/gcp_triggers/topics/list/{path}: + get: + summary: list all topics of google cloud service + operationId: listGoogleTopics + tags: + - gcp_trigger + parameters: + - name: workspace + in: path + required: true + schema: *ref_0 + - name: path + in: path + required: true + schema: *ref_26 + responses: + '200': + description: get all google topics + content: + application/json: + schema: + type: array + items: + type: string + /w/{workspace}/gcp_triggers/subscriptions/list/{path}: + post: + summary: list all subscription of a give topic from google cloud service + operationId: listAllTGoogleTopicSubscriptions + tags: + - gcp_trigger + parameters: + - name: workspace + in: path + required: true + schema: *ref_0 + - name: path + in: path + required: true + schema: *ref_26 + requestBody: + description: args to get subscription's topic from google cloud + required: true + content: + application/json: + schema: + type: object + properties: &ref_253 + topic_id: + type: string + required: &ref_254 + - topic_id + responses: + '200': + description: get all google topic subscriptions name + content: + application/json: + schema: + type: array + items: + type: string + /w/{workspace}/postgres_triggers/is_valid_postgres_configuration/{path}: + get: + summary: check if postgres configuration is set to logical + operationId: isValidPostgresConfiguration + tags: + - postgres_trigger + parameters: + - name: workspace + in: path + required: true + schema: *ref_0 + - name: path + in: path + required: true + schema: *ref_26 + responses: + '200': + description: boolean that indicates if postgres is set to logical level or not + content: + application/json: + schema: + type: boolean + /w/{workspace}/postgres_triggers/create_template_script: + post: + summary: create template script + operationId: createTemplateScript + tags: + - postgres_trigger + parameters: + - name: workspace + in: path + required: true + schema: *ref_0 + requestBody: + description: template script + required: true + content: + application/json: + schema: + type: object + properties: &ref_264 + postgres_resource_path: + type: string + relations: + type: array + items: + type: object + properties: &ref_155 + schema_name: + type: string + table_to_track: + type: array + items: &ref_262 + type: object + properties: + table_name: + type: string + columns_name: + type: array + items: + type: string + where_clause: + type: string + required: + - table_name + required: &ref_156 + - schema_name + - table_to_track + language: + type: string + enum: &ref_263 + - Typescript + required: &ref_265 + - postgres_resource_path + - relations + - language + responses: + '200': + description: custom id to retrieve template script + content: + text/plain: + schema: + type: string + /w/{workspace}/postgres_triggers/get_template_script/{id}: + get: + summary: get template script + operationId: getTemplateScript + tags: + - postgres_trigger + parameters: + - name: workspace + in: path + required: true + schema: *ref_0 + - name: id + in: path + required: true + schema: &ref_185 + type: string + responses: + '200': + description: template script + content: + text/plain: + schema: + type: string + /w/{workspace}/postgres_triggers/slot/list/{path}: + get: + summary: list postgres replication slot + operationId: listPostgresReplicationSlot + tags: + - postgres_trigger + parameters: + - name: workspace + in: path + required: true + schema: *ref_0 + - name: path + in: path + required: true + schema: *ref_26 + responses: + '200': + description: list postgres slot + content: + application/json: + schema: + type: array + items: + type: object + properties: &ref_261 + slot_name: + type: string + active: + type: boolean + /w/{workspace}/postgres_triggers/slot/create/{path}: + post: + summary: create replication slot for postgres + operationId: createPostgresReplicationSlot + tags: + - postgres_trigger + parameters: + - name: workspace + in: path + required: true + schema: *ref_0 + - name: path + in: path + required: true + schema: *ref_26 + requestBody: + description: new slot for postgres + required: true + content: + application/json: + schema: + type: object + properties: &ref_154 + name: + type: string + responses: + '201': + description: slot created + content: + text/plain: + schema: + type: string + /w/{workspace}/postgres_triggers/slot/delete/{path}: + delete: + summary: delete postgres replication slot + operationId: deletePostgresReplicationSlot + tags: + - postgres_trigger + parameters: + - name: workspace + in: path + required: true + schema: *ref_0 + - name: path + in: path + required: true + schema: *ref_26 + requestBody: + description: replication slot of postgres + required: true + content: + application/json: + schema: + type: object + properties: *ref_154 + responses: + '200': + description: postgres replication slot deleted + content: + text/plain: + schema: + type: string + /w/{workspace}/postgres_triggers/publication/list/{path}: + get: + summary: list postgres publication + operationId: listPostgresPublication + tags: + - postgres_trigger + parameters: + - name: workspace + in: path + required: true + schema: *ref_0 + - name: path + in: path + required: true + schema: *ref_26 + responses: + '200': + description: database publication list + content: + application/json: + schema: + type: array + items: + type: string + /w/{workspace}/postgres_triggers/publication/get/{publication}/{path}: + get: + summary: get postgres publication + operationId: getPostgresPublication + tags: + - postgres_trigger + parameters: + - name: workspace + in: path + required: true + schema: *ref_0 + - name: path + in: path + required: true + schema: *ref_26 + - name: publication + in: path + required: true + schema: &ref_157 + type: string + responses: + '200': + description: postgres publication get + content: + application/json: + schema: + type: object + properties: &ref_158 + table_to_track: + type: array + items: + type: object + properties: *ref_155 + required: *ref_156 + transaction_to_track: + type: array + items: + type: string + required: &ref_159 + - transaction_to_track + /w/{workspace}/postgres_triggers/publication/create/{publication}/{path}: + post: + summary: create publication for postgres + operationId: createPostgresPublication + tags: + - postgres_trigger + parameters: + - name: workspace + in: path + required: true + schema: *ref_0 + - name: path + in: path + required: true + schema: *ref_26 + - name: publication + in: path + required: true + schema: *ref_157 + requestBody: + description: new publication for postgres + required: true + content: + application/json: + schema: + type: object + properties: *ref_158 + required: *ref_159 + responses: + '201': + description: publication created + content: + text/plain: + schema: + type: string + /w/{workspace}/postgres_triggers/publication/update/{publication}/{path}: + post: + summary: update publication for postgres + operationId: updatePostgresPublication + tags: + - postgres_trigger + parameters: + - name: workspace + in: path + required: true + schema: *ref_0 + - name: path + in: path + required: true + schema: *ref_26 + - name: publication + in: path + required: true + schema: *ref_157 + requestBody: + description: update publication for postgres + required: true + content: + application/json: + schema: + type: object + properties: *ref_158 + required: *ref_159 + responses: + '201': + description: publication updated + content: + text/plain: + schema: + type: string + /w/{workspace}/postgres_triggers/publication/delete/{publication}/{path}: + delete: + summary: delete postgres publication + operationId: deletePostgresPublication + tags: + - postgres_trigger + parameters: + - name: workspace + in: path + required: true + schema: *ref_0 + - name: path + in: path + required: true + schema: *ref_26 + - name: publication + in: path + required: true + schema: *ref_157 + responses: + '200': + description: postgres publication deleted + content: + text/plain: + schema: + type: string + /w/{workspace}/postgres_triggers/create: + post: + summary: create postgres trigger + operationId: createPostgresTrigger + tags: + - postgres_trigger + parameters: + - name: workspace + in: path + required: true + schema: *ref_0 + requestBody: + description: new postgres trigger + required: true + content: + application/json: + schema: + type: object + properties: &ref_266 + replication_slot_name: + type: string + publication_name: + type: string + path: + type: string + script_path: + type: string + is_flow: + type: boolean + enabled: + type: boolean + postgres_resource_path: + type: string + publication: + type: object + properties: *ref_158 + required: *ref_159 + required: &ref_267 + - path + - script_path + - is_flow + - enabled + - postgres_resource_path + responses: + '201': + description: postgres trigger created + content: + text/plain: + schema: + type: string + /w/{workspace}/postgres_triggers/update/{path}: + post: + summary: update postgres trigger + operationId: updatePostgresTrigger + tags: + - postgres_trigger + parameters: + - name: workspace + in: path + required: true + schema: *ref_0 + - name: path + in: path + required: true + schema: *ref_26 + requestBody: + description: updated trigger + required: true + content: + application/json: + schema: + type: object + properties: &ref_268 + replication_slot_name: + type: string + publication_name: + type: string + path: + type: string + script_path: + type: string + is_flow: + type: boolean + enabled: + type: boolean + postgres_resource_path: + type: string + publication: + type: object + properties: *ref_158 + required: *ref_159 + required: &ref_269 + - path + - script_path + - is_flow + - enabled + - postgres_resource_path + - publication_name + - replication_slot_name + responses: + '200': + description: postgres trigger updated + content: + text/plain: + schema: + type: string + /w/{workspace}/postgres_triggers/delete/{path}: + delete: + summary: delete postgres trigger + operationId: deletePostgresTrigger + tags: + - postgres_trigger + parameters: + - name: workspace + in: path + required: true + schema: *ref_0 + - name: path + in: path + required: true + schema: *ref_26 + responses: + '200': + description: postgres trigger deleted + content: + text/plain: + schema: + type: string + /w/{workspace}/postgres_triggers/get/{path}: + get: + summary: get postgres trigger + operationId: getPostgresTrigger + tags: + - postgres_trigger + parameters: + - name: workspace + in: path + required: true + schema: *ref_0 + - name: path + in: path + required: true + schema: *ref_26 + responses: + '200': + description: get postgres trigger + content: + application/json: + schema: + allOf: &ref_160 + - type: object + properties: *ref_122 + required: *ref_123 + type: object + properties: &ref_161 + enabled: + type: boolean + postgres_resource_path: + type: string + publication_name: + type: string + server_id: + type: string + replication_slot_name: + type: string + error: + type: string + last_server_ping: + type: string + format: date-time + required: &ref_162 + - enabled + - postgres_resource_path + - replication_slot_name + - publication_name + /w/{workspace}/postgres_triggers/list: + get: + summary: list postgres triggers + operationId: listPostgresTriggers + tags: + - postgres_trigger + parameters: + - required: true + name: workspace + in: path + schema: *ref_0 + - name: page + description: which page to return (start at 1, default 1) + in: query + schema: *ref_5 + - name: per_page + description: number of items to return for a given page (default 30, max 100) + in: query + schema: *ref_6 + - name: path + description: filter by path + in: query + schema: + type: string + - name: is_flow + in: query + schema: + type: boolean + - name: path_start + in: query + schema: + type: string + responses: + '200': + description: postgres trigger list + content: + application/json: + schema: + type: array + items: + allOf: *ref_160 + type: object + properties: *ref_161 + required: *ref_162 + /w/{workspace}/postgres_triggers/exists/{path}: + get: + summary: does postgres trigger exists + operationId: existsPostgresTrigger + tags: + - postgres_trigger + parameters: + - name: workspace + in: path + required: true + schema: *ref_0 + - name: path + in: path + required: true + schema: *ref_26 + responses: + '200': + description: postgres trigger exists + content: + application/json: + schema: + type: boolean + /w/{workspace}/postgres_triggers/setenabled/{path}: + post: + summary: set enabled postgres trigger + operationId: setPostgresTriggerEnabled + tags: + - postgres_trigger + parameters: + - name: workspace + in: path + required: true + schema: *ref_0 + - name: path + in: path + required: true + schema: *ref_26 + requestBody: + description: updated postgres trigger enable + required: true + content: + application/json: + schema: + type: object + properties: + enabled: + type: boolean + required: + - enabled + responses: + '200': + description: postgres trigger enabled set + content: + text/plain: + schema: + type: string + /w/{workspace}/postgres_triggers/test: + post: + summary: test postgres connection + operationId: testPostgresConnection + tags: + - postgres_trigger + parameters: + - name: workspace + in: path + required: true + schema: *ref_0 + requestBody: + description: test postgres connection + required: true + content: + application/json: + schema: + type: object + properties: + database: + type: string + required: + - database + responses: + '200': + description: successfuly connected to postgres + content: + text/plain: + schema: + type: string /groups/list: get: summary: list instance groups @@ -11601,7 +15683,7 @@ paths: type: array items: type: object - properties: &ref_104 + properties: &ref_164 name: type: string summary: @@ -11610,7 +15692,7 @@ paths: type: array items: type: string - required: &ref_105 + required: &ref_165 - name /groups/get/{name}: get: @@ -11622,7 +15704,7 @@ paths: - name: name in: path required: true - schema: *ref_103 + schema: *ref_163 responses: '200': description: instance group @@ -11630,8 +15712,8 @@ paths: application/json: schema: type: object - properties: *ref_104 - required: *ref_105 + properties: *ref_164 + required: *ref_165 /groups/create: post: summary: create instance group @@ -11669,7 +15751,7 @@ paths: - name: name in: path required: true - schema: *ref_103 + schema: *ref_163 requestBody: description: update instance group required: true @@ -11699,7 +15781,7 @@ paths: - name: name in: path required: true - schema: *ref_103 + schema: *ref_163 responses: '200': description: instance group deleted @@ -11717,7 +15799,7 @@ paths: - name: name in: path required: true - schema: *ref_103 + schema: *ref_163 requestBody: description: user to add to instance group required: true @@ -11747,7 +15829,7 @@ paths: - name: name in: path required: true - schema: *ref_103 + schema: *ref_163 requestBody: description: user to remove from instance group required: true @@ -11782,7 +15864,7 @@ paths: type: array items: type: object - properties: &ref_106 + properties: &ref_166 name: type: string summary: @@ -11797,7 +15879,7 @@ paths: type: string external_id: type: string - required: &ref_107 + required: &ref_167 - name /groups/overwrite: post: @@ -11814,8 +15896,8 @@ paths: type: array items: type: object - properties: *ref_106 - required: *ref_107 + properties: *ref_166 + required: *ref_167 responses: '200': description: success message @@ -11851,7 +15933,7 @@ paths: type: array items: type: object - properties: &ref_108 + properties: &ref_168 name: type: string summary: @@ -11864,7 +15946,7 @@ paths: type: object additionalProperties: type: boolean - required: &ref_109 + required: &ref_169 - name /w/{workspace}/groups/listnames: get: @@ -11937,7 +16019,7 @@ paths: - name: name in: path required: true - schema: *ref_103 + schema: *ref_163 requestBody: description: updated group required: true @@ -11969,7 +16051,7 @@ paths: - name: name in: path required: true - schema: *ref_103 + schema: *ref_163 responses: '200': description: group deleted @@ -11991,7 +16073,7 @@ paths: - name: name in: path required: true - schema: *ref_103 + schema: *ref_163 responses: '200': description: group @@ -11999,8 +16081,8 @@ paths: application/json: schema: type: object - properties: *ref_108 - required: *ref_109 + properties: *ref_168 + required: *ref_169 /w/{workspace}/groups/adduser/{name}: post: summary: add user to group @@ -12015,7 +16097,7 @@ paths: - name: name in: path required: true - schema: *ref_103 + schema: *ref_163 requestBody: description: added user to group required: true @@ -12047,7 +16129,7 @@ paths: - name: name in: path required: true - schema: *ref_103 + schema: *ref_163 requestBody: description: added user to group required: true @@ -12093,7 +16175,7 @@ paths: type: array items: type: object - properties: &ref_110 + properties: &ref_170 name: type: string owners: @@ -12111,7 +16193,7 @@ paths: edited_at: type: string format: date-time - required: &ref_111 + required: &ref_171 - name - owners - extra_perms @@ -12193,7 +16275,7 @@ paths: - name: name in: path required: true - schema: *ref_103 + schema: *ref_163 requestBody: description: update folder required: true @@ -12232,7 +16314,7 @@ paths: - name: name in: path required: true - schema: *ref_103 + schema: *ref_163 responses: '200': description: folder deleted @@ -12254,7 +16336,7 @@ paths: - name: name in: path required: true - schema: *ref_103 + schema: *ref_163 responses: '200': description: folder @@ -12262,8 +16344,30 @@ paths: application/json: schema: type: object - properties: *ref_110 - required: *ref_111 + properties: *ref_170 + required: *ref_171 + /w/{workspace}/folders/exists/{name}: + get: + summary: exists folder + operationId: existsFolder + tags: + - folder + parameters: + - name: workspace + in: path + required: true + schema: *ref_0 + - name: name + in: path + required: true + schema: *ref_163 + responses: + '200': + description: folder exists + content: + application/json: + schema: + type: boolean /w/{workspace}/folders/getusage/{name}: get: summary: get folder usage @@ -12278,7 +16382,7 @@ paths: - name: name in: path required: true - schema: *ref_103 + schema: *ref_163 responses: '200': description: folder @@ -12320,7 +16424,7 @@ paths: - name: name in: path required: true - schema: *ref_103 + schema: *ref_163 requestBody: description: owner user to folder required: true @@ -12354,7 +16458,7 @@ paths: - name: name in: path required: true - schema: *ref_103 + schema: *ref_163 requestBody: description: added owner to folder required: true @@ -12408,7 +16512,7 @@ paths: type: array items: type: object - properties: &ref_183 + properties: &ref_278 worker: type: string worker_instance: @@ -12450,7 +16554,7 @@ paths: type: number wm_memory_usage: type: number - required: &ref_184 + required: &ref_279 - worker - worker_instance - ping_at @@ -12558,13 +16662,39 @@ paths: - name: name in: path required: true - schema: *ref_103 + schema: *ref_163 responses: '200': description: a config content: application/json: - schema: {} + schema: + type: object + nullable: true + properties: &ref_201 + alerts: + type: array + items: + type: object + properties: &ref_199 + name: + type: string + tags_to_monitor: + type: array + items: + type: string + jobs_num_threshold: + type: integer + alert_cooldown_seconds: + type: integer + alert_time_threshold_seconds: + type: integer + required: &ref_200 + - name + - tags_to_monitor + - jobs_num_threshold + - alert_cooldown_seconds + - alert_time_threshold_seconds /configs/update/{name}: post: summary: Update config @@ -12575,7 +16705,7 @@ paths: - name: name in: path required: true - schema: *ref_103 + schema: *ref_163 requestBody: description: worker group required: true @@ -12598,7 +16728,7 @@ paths: - name: name in: path required: true - schema: *ref_103 + schema: *ref_163 responses: '200': description: Delete config @@ -12621,12 +16751,12 @@ paths: type: array items: type: object - properties: &ref_225 + properties: &ref_320 name: type: string config: type: object - required: &ref_226 + required: &ref_321 - name /configs/list_autoscaling_events/{worker_group}: get: @@ -12649,7 +16779,7 @@ paths: type: array items: type: object - properties: &ref_229 + properties: &ref_324 id: type: integer format: int64 @@ -12664,6 +16794,54 @@ paths: applied_at: type: string format: date-time + /configs/list_available_python_versions: + get: + summary: Get currently available python versions provided by UV. + operationId: listAvailablePythonVersions + tags: + - config + responses: + '200': + description: List of python versions + content: + application/json: + schema: + type: array + items: + type: string + /agent_workers/create_agent_token: + post: + summary: create agent token + operationId: createAgentToken + tags: + - agent_workers + requestBody: + description: agent token + required: true + content: + application/json: + schema: + type: object + properties: + worker_group: + type: string + tags: + type: array + items: + type: string + exp: + type: integer + required: + - worker_group + - tags + - exp + responses: + '200': + description: agent token created + content: + application/json: + schema: + type: string /w/{workspace}/acls/get/{kind}/{path}: get: summary: get granular acls @@ -12678,7 +16856,7 @@ paths: - name: path in: path required: true - schema: *ref_23 + schema: *ref_26 - name: kind in: path required: true @@ -12696,6 +16874,12 @@ paths: - raw_app - http_trigger - websocket_trigger + - kafka_trigger + - nats_trigger + - postgres_trigger + - mqtt_trigger + - gcp_trigger + - sqs_trigger responses: '200': description: acls @@ -12719,7 +16903,7 @@ paths: - name: path in: path required: true - schema: *ref_23 + schema: *ref_26 - name: kind in: path required: true @@ -12737,6 +16921,12 @@ paths: - raw_app - http_trigger - websocket_trigger + - kafka_trigger + - nats_trigger + - postgres_trigger + - mqtt_trigger + - gcp_trigger + - sqs_trigger requestBody: description: acl to add required: true @@ -12772,7 +16962,7 @@ paths: - name: path in: path required: true - schema: *ref_23 + schema: *ref_26 - name: kind in: path required: true @@ -12790,6 +16980,12 @@ paths: - raw_app - http_trigger - websocket_trigger + - kafka_trigger + - nats_trigger + - postgres_trigger + - mqtt_trigger + - gcp_trigger + - sqs_trigger requestBody: description: acl to add required: true @@ -12809,10 +17005,10 @@ paths: text/plain: schema: type: string - /w/{workspace}/capture_u/{path}: + /w/{workspace}/capture/set_config: post: - summary: update flow preview capture - operationId: updateCapture + summary: set capture config + operationId: setCaptureConfig tags: - capture parameters: @@ -12820,17 +17016,48 @@ paths: in: path required: true schema: *ref_0 - - name: path - in: path - required: true - schema: *ref_23 + requestBody: + description: capture config + required: true + content: + application/json: + schema: + type: object + properties: + trigger_kind: + type: string + enum: &ref_172 + - webhook + - http + - websocket + - kafka + - email + - nats + - postgres + - sqs + - mqtt + - gcp + path: + type: string + is_flow: + type: boolean + trigger_config: + type: object + required: + - trigger_kind + - path + - is_flow responses: - '204': - description: flow preview captured - /w/{workspace}/capture/{path}: - put: - summary: create flow preview capture - operationId: createCapture + '200': + description: capture config set + content: + application/json: + schema: + type: object + /w/{workspace}/capture/ping_config/{trigger_kind}/{runnable_kind}/{path}: + post: + summary: ping capture config + operationId: pingCaptureConfig tags: - capture parameters: @@ -12838,15 +17065,160 @@ paths: in: path required: true schema: *ref_0 + - name: trigger_kind + in: path + required: true + schema: + type: string + enum: *ref_172 + - name: runnable_kind + in: path + required: true + schema: *ref_73 - name: path in: path required: true - schema: *ref_23 + schema: *ref_26 responses: - '201': - description: flow preview capture created + '200': + description: capture config pinged + /w/{workspace}/capture/get_configs/{runnable_kind}/{path}: get: - summary: get flow preview capture + summary: get capture configs for a script or flow + operationId: getCaptureConfigs + tags: + - capture + parameters: + - name: workspace + in: path + required: true + schema: *ref_0 + - name: runnable_kind + in: path + required: true + schema: *ref_73 + - name: path + in: path + required: true + schema: *ref_26 + responses: + '200': + description: capture configs for a script or flow + content: + application/json: + schema: + type: array + items: + type: object + properties: &ref_325 + trigger_config: {} + trigger_kind: + type: string + enum: *ref_172 + error: + type: string + last_server_ping: + type: string + format: date-time + required: &ref_326 + - trigger_kind + /w/{workspace}/capture/list/{runnable_kind}/{path}: + get: + summary: list captures for a script or flow + operationId: listCaptures + tags: + - capture + parameters: + - name: workspace + in: path + required: true + schema: *ref_0 + - name: runnable_kind + in: path + required: true + schema: *ref_73 + - name: path + in: path + required: true + schema: *ref_26 + - name: trigger_kind + in: query + schema: + type: string + enum: *ref_172 + - name: page + description: which page to return (start at 1, default 1) + in: query + schema: *ref_5 + - name: per_page + description: number of items to return for a given page (default 30, max 100) + in: query + schema: *ref_6 + responses: + '200': + description: list of captures for a script or flow + content: + application/json: + schema: + type: array + items: + type: object + properties: &ref_173 + trigger_kind: + type: string + enum: *ref_172 + main_args: {} + preprocessor_args: {} + id: + type: integer + created_at: + type: string + format: date-time + required: &ref_174 + - trigger_kind + - main_args + - preprocessor_args + - id + - created_at + /w/{workspace}/capture/move/{runnable_kind}/{path}: + post: + summary: move captures and configs for a script or flow + operationId: moveCapturesAndConfigs + tags: + - capture + parameters: + - name: workspace + in: path + required: true + schema: *ref_0 + - name: runnable_kind + in: path + required: true + schema: *ref_73 + - name: path + in: path + required: true + schema: *ref_26 + requestBody: + description: move captures and configs to a new path + required: true + content: + application/json: + schema: + type: object + properties: + new_path: + type: string + responses: + '200': + description: captures and configs moved + content: + text/plain: + schema: + type: string + /w/{workspace}/capture/{id}: + get: + summary: get a capture operationId: getCapture tags: - capture @@ -12855,18 +17227,38 @@ paths: in: path required: true schema: *ref_0 - - name: path + - name: id in: path required: true - schema: *ref_23 + schema: + type: integer responses: '200': - description: captured flow preview + description: capture content: application/json: - schema: {} - '404': - description: capture does not exist for this flow + schema: + type: object + properties: *ref_173 + required: *ref_174 + delete: + summary: delete a capture + operationId: deleteCapture + tags: + - capture + parameters: + - name: workspace + in: path + required: true + schema: *ref_0 + - name: id + in: path + required: true + schema: + type: integer + responses: + '200': + description: capture deleted /w/{workspace}/favorites/star: post: summary: star item @@ -12938,13 +17330,13 @@ paths: schema: *ref_0 - name: runnable_id in: query - schema: &ref_112 + schema: &ref_175 type: string - name: runnable_type in: query - schema: &ref_113 + schema: &ref_176 type: string - enum: &ref_142 + enum: &ref_207 - ScriptHash - ScriptPath - FlowPath @@ -12956,6 +17348,16 @@ paths: description: number of items to return for a given page (default 30, max 100) in: query schema: *ref_6 + - name: args + description: >- + filter on jobs containing those args as a json subset (@> in + postgres) + in: query + schema: *ref_93 + - name: include_preview + in: query + schema: + type: boolean responses: '200': description: Input history for completed jobs @@ -12965,7 +17367,7 @@ paths: type: array items: type: object - properties: &ref_114 + properties: &ref_177 id: type: string name: @@ -12979,7 +17381,7 @@ paths: type: boolean success: type: boolean - required: &ref_115 + required: &ref_178 - id - name - args @@ -13029,10 +17431,10 @@ paths: schema: *ref_0 - name: runnable_id in: query - schema: *ref_112 + schema: *ref_175 - name: runnable_type in: query - schema: *ref_113 + schema: *ref_176 - name: page description: which page to return (start at 1, default 1) in: query @@ -13050,8 +17452,8 @@ paths: type: array items: type: object - properties: *ref_114 - required: *ref_115 + properties: *ref_177 + required: *ref_178 /w/{workspace}/inputs/create: post: summary: Create an Input for future use in a script or flow @@ -13065,10 +17467,10 @@ paths: schema: *ref_0 - name: runnable_id in: query - schema: *ref_112 + schema: *ref_175 - name: runnable_type in: query - schema: *ref_113 + schema: *ref_176 requestBody: description: Input required: true @@ -13076,12 +17478,12 @@ paths: application/json: schema: type: object - properties: &ref_138 + properties: &ref_203 name: type: string args: type: object - required: &ref_139 + required: &ref_204 - name - args - created_by @@ -13111,14 +17513,14 @@ paths: application/json: schema: type: object - properties: &ref_140 + properties: &ref_205 id: type: string name: type: string is_public: type: boolean - required: &ref_141 + required: &ref_206 - id - name - is_public @@ -13144,7 +17546,7 @@ paths: - name: input in: path required: true - schema: &ref_134 + schema: &ref_193 type: string responses: '200': @@ -13177,7 +17579,7 @@ paths: properties: s3_resource: type: object - properties: &ref_116 + properties: &ref_179 bucket: type: string region: @@ -13192,7 +17594,7 @@ paths: type: string pathStyle: type: boolean - required: &ref_117 + required: &ref_180 - bucket - region - endPoint @@ -13243,6 +17645,8 @@ paths: properties: connection_settings_str: type: string + azure_container_path: + type: string required: - connection_settings_str /w/{workspace}/job_helpers/polars_connection_settings: @@ -13268,8 +17672,8 @@ paths: properties: s3_resource: type: object - properties: *ref_116 - required: *ref_117 + properties: *ref_179 + required: *ref_180 responses: '200': description: Connection settings @@ -13290,10 +17694,10 @@ paths: type: boolean client_kwargs: type: object - properties: &ref_118 + properties: &ref_181 region_name: type: string - required: &ref_119 + required: &ref_182 - region_name required: - endpoint_url @@ -13348,8 +17752,8 @@ paths: type: boolean client_kwargs: type: object - properties: *ref_118 - required: *ref_119 + properties: *ref_181 + required: *ref_182 required: - endpoint_url - use_ssl @@ -13407,8 +17811,8 @@ paths: application/json: schema: type: object - properties: *ref_116 - required: *ref_117 + properties: *ref_179 + required: *ref_180 /w/{workspace}/job_helpers/test_connection: get: summary: Test connection to the workspace object storage @@ -13472,10 +17876,10 @@ paths: type: array items: type: object - properties: &ref_202 + properties: &ref_297 s3: type: string - required: &ref_203 + required: &ref_298 - s3 restricted_access: type: boolean @@ -13508,7 +17912,7 @@ paths: application/json: schema: type: object - properties: &ref_204 + properties: &ref_299 mime_type: type: string size_in_bytes: @@ -13572,7 +17976,7 @@ paths: application/json: schema: type: object - properties: &ref_205 + properties: &ref_300 msg: type: string content: @@ -13584,7 +17988,7 @@ paths: - Csv - Parquet - Unknown - required: &ref_206 + required: &ref_301 - content_type /w/{workspace}/job_helpers/load_parquet_preview/{path}: get: @@ -13600,7 +18004,7 @@ paths: - name: path in: path required: true - schema: *ref_23 + schema: *ref_26 - name: offset in: query schema: @@ -13649,7 +18053,7 @@ paths: - name: path in: path required: true - schema: *ref_23 + schema: *ref_26 - name: search_col in: query schema: @@ -13686,7 +18090,7 @@ paths: - name: path in: path required: true - schema: *ref_23 + schema: *ref_26 - name: offset in: query schema: @@ -13847,7 +18251,7 @@ paths: - file_key /w/{workspace}/job_helpers/download_s3_file: get: - summary: Download file to S3 bucket + summary: Download file from S3 bucket operationId: fileDownload tags: - helpers @@ -13930,7 +18334,7 @@ paths: - name: id in: path required: true - schema: *ref_91 + schema: *ref_110 requestBody: description: parameters for statistics retrieval required: true @@ -13959,46 +18363,46 @@ paths: type: array items: type: object - properties: &ref_209 + properties: &ref_304 id: type: string name: type: string - required: &ref_210 + required: &ref_305 - id scalar_metrics: type: array items: type: object - properties: &ref_211 + properties: &ref_306 metric_id: type: string value: type: number - required: &ref_212 + required: &ref_307 - id - value timeseries_metrics: type: array items: type: object - properties: &ref_213 + properties: &ref_308 metric_id: type: string values: type: array items: type: object - properties: &ref_215 + properties: &ref_310 timestamp: type: string format: date-time value: type: number - required: &ref_216 + required: &ref_311 - timestamp - value - required: &ref_214 + required: &ref_309 - id - values /w/{workspace}/job_metrics/set_progress/{id}: @@ -14015,7 +18419,7 @@ paths: - name: id in: path required: true - schema: *ref_91 + schema: *ref_110 requestBody: description: parameters for statistics retrieval required: true @@ -14049,7 +18453,7 @@ paths: - name: id in: path required: true - schema: *ref_91 + schema: *ref_110 responses: '200': description: job progress between 0 and 99 @@ -14067,11 +18471,11 @@ paths: - name: before description: filter on started before (inclusive) timestamp in: query - schema: *ref_120 + schema: *ref_183 - name: after description: filter on created after (exclusive) timestamp in: query - schema: *ref_121 + schema: *ref_184 - name: with_error in: query required: false @@ -14120,7 +18524,7 @@ paths: - name: path in: path required: true - schema: *ref_23 + schema: *ref_26 responses: '200': description: log stream @@ -14143,12 +18547,12 @@ paths: type: array items: type: object - properties: &ref_219 + properties: &ref_314 concurrency_key: type: string total_running: type: number - required: &ref_220 + required: &ref_315 - concurrency_key - total_running /concurrency_groups/prune/{concurrency_id}: @@ -14161,7 +18565,7 @@ paths: - name: concurrency_id in: path required: true - schema: &ref_136 + schema: &ref_195 type: string responses: '200': @@ -14181,7 +18585,7 @@ paths: - name: id in: path required: true - schema: *ref_91 + schema: *ref_110 responses: '200': description: concurrency key for given job @@ -14214,93 +18618,97 @@ paths: - name: created_by description: mask to filter exact matching user creator in: query - schema: *ref_53 + schema: *ref_60 - name: label description: >- mask to filter exact matching job's label (job labels are completed jobs with as a result an object containing a string in the array at key 'wm_labels') in: query - schema: *ref_86 + schema: *ref_98 - name: parent_job description: >- The parent job that is at the origin and responsible for the execution of this script if any in: query - schema: *ref_46 + schema: *ref_53 - name: script_path_exact description: mask to filter exact matching path in: query - schema: *ref_70 + schema: *ref_83 - name: script_path_start description: mask to filter matching starting path in: query - schema: *ref_71 + schema: *ref_84 - name: schedule_path description: mask to filter by schedule path in: query - schema: *ref_72 + schema: *ref_85 - name: script_hash description: mask to filter exact matching path in: query - schema: *ref_73 + schema: *ref_86 - name: started_before description: filter on started before (inclusive) timestamp in: query - schema: *ref_74 + schema: *ref_87 - name: started_after description: filter on started after (exclusive) timestamp in: query - schema: *ref_75 + schema: *ref_88 - name: created_or_started_before description: >- filter on created_at for non non started job and started_at otherwise before (inclusive) timestamp in: query - schema: *ref_122 + schema: *ref_103 - name: running description: filter on running jobs in: query - schema: *ref_80 + schema: *ref_89 - name: scheduled_for_before_now description: filter on jobs scheduled_for before now (hence waitinf for a worker) in: query - schema: *ref_77 + schema: *ref_90 - name: created_or_started_after description: >- filter on created_at for non non started job and started_at otherwise after (exclusive) timestamp in: query - schema: *ref_123 + schema: *ref_104 - name: created_or_started_after_completed_jobs description: >- filter on created_at for non non started job and started_at otherwise after (exclusive) timestamp but only for the completed jobs in: query - schema: *ref_124 + schema: *ref_105 - name: job_kinds description: >- filter on job kind (values 'preview', 'script', 'dependencies', 'flow') separated by, in: query - schema: *ref_78 + schema: *ref_91 - name: args description: >- filter on jobs containing those args as a json subset (@> in postgres) in: query - schema: *ref_81 + schema: *ref_93 - name: tag description: filter on jobs with a given tag/worker group in: query - schema: *ref_83 + schema: *ref_94 - name: result description: >- filter on jobs containing those result as a json subset (@> in postgres) in: query - schema: *ref_82 + schema: *ref_95 + - name: allow_wildcards + description: allow wildcards (*) in the filter of label, tag, worker + in: query + schema: *ref_97 - name: page description: which page to return (start at 1, default 1) in: query @@ -14348,17 +18756,17 @@ paths: application/json: schema: type: object - properties: &ref_221 + properties: &ref_316 jobs: type: array items: - oneOf: *ref_92 - discriminator: *ref_93 + oneOf: *ref_111 + discriminator: *ref_112 obscured_jobs: type: array items: type: object - properties: &ref_143 + properties: &ref_208 typ: type: string started_at: @@ -14371,7 +18779,7 @@ paths: Obscured jobs omitted for security because of too specific filtering type: boolean - required: &ref_222 + required: &ref_317 - jobs - obscured_jobs /srch/w/{workspace}/index/search/job: @@ -14390,6 +18798,11 @@ paths: required: true schema: type: string + - name: pagination_offset + in: query + required: false + schema: + type: integer responses: '200': description: search results @@ -14404,18 +18817,29 @@ paths: ignored) type: array items: - type: object - properties: - dancer: - type: string + type: string hits: description: the jobs that matched the query type: array items: type: object - properties: &ref_227 + properties: &ref_322 dancer: type: string + hit_count: + description: how many jobs matched in total + type: number + index_metadata: + description: Metadata about the index current state + type: object + properties: + indexed_until: + description: Datetime of the most recently indexed job + type: string + format: date-time + lost_lock_ownership: + description: Is the current indexer service being replaced + type: boolean /srch/index/search/service_logs: get: summary: Search through service logs with a string query @@ -14475,7 +18899,7 @@ paths: type: array items: type: object - properties: &ref_228 + properties: &ref_323 dancer: type: string /srch/index/search/count_service_logs: @@ -14490,11 +18914,6 @@ paths: required: true schema: type: string - - name: hosts - in: query - required: true - schema: - type: string - name: min_ts in: query required: false @@ -14525,6 +18944,28 @@ paths: count_per_host: description: count of log lines that matched the query per hostname type: object + /srch/index/delete/{idx_name}: + delete: + summary: Restart container and delete the index to recreate it. + operationId: clearIndex + tags: + - indexSearch + parameters: + - name: idx_name + in: path + required: true + schema: + type: string + enum: + - JobIndex + - ServiceLogIndex + responses: + '200': + description: idx to be deleted and container restarting + content: + text/plain: + schema: + type: string components: securitySchemes: bearerAuth: @@ -14535,6 +18976,11 @@ components: in: cookie name: token parameters: + Id: + name: id + in: path + required: true + schema: *ref_185 Key: name: key in: path @@ -14545,61 +18991,71 @@ components: in: path required: true schema: *ref_0 + PublicationName: + name: publication + in: path + required: true + schema: *ref_157 VersionId: name: version in: path required: true - schema: *ref_125 + schema: *ref_186 Token: name: token in: path required: true - schema: *ref_126 + schema: *ref_187 AccountId: name: id in: path required: true - schema: *ref_27 + schema: *ref_31 ClientName: name: client_name in: path required: true - schema: *ref_26 + schema: *ref_30 ScriptPath: name: path in: path required: true - schema: *ref_35 + schema: *ref_41 ScriptHash: name: hash in: path required: true - schema: *ref_39 + schema: *ref_46 JobId: name: id in: path required: true - schema: *ref_91 + schema: *ref_110 Path: name: path in: path required: true - schema: *ref_23 + schema: *ref_26 + CustomPath: + name: custom_path + in: path + required: true + schema: *ref_74 PathId: name: id in: path required: true - schema: *ref_30 + schema: *ref_34 PathVersion: name: version in: path required: true - schema: *ref_127 + schema: *ref_188 Name: name: name in: path required: true - schema: *ref_103 + schema: *ref_163 Page: name: page description: which page to return (start at 1, default 1) @@ -14614,12 +19070,12 @@ components: name: order_desc description: order by desc order (default true) in: query - schema: *ref_52 + schema: *ref_59 CreatedBy: name: created_by description: mask to filter exact matching user creator in: query - schema: *ref_53 + schema: *ref_60 Label: name: label description: >- @@ -14627,26 +19083,31 @@ components: with as a result an object containing a string in the array at key 'wm_labels') in: query - schema: *ref_86 + schema: *ref_98 + Worker: + name: worker + description: worker this job was ran on + in: query + schema: *ref_82 ParentJob: name: parent_job description: >- The parent job that is at the origin and responsible for the execution of this script if any in: query - schema: *ref_46 + schema: *ref_53 WorkerTag: name: tag description: Override the tag to use in: query - schema: *ref_48 + schema: *ref_55 CacheTtl: name: cache_ttl description: >- Override the cache time to live (in seconds). Can not be used to disable caching, only override with a new cache ttl in: query - schema: *ref_49 + schema: *ref_56 NewJobId: name: job_id description: >- @@ -14654,7 +19115,7 @@ components: randomly using the ULID scheme. If a job id already exists in the queue or as a completed job, the request to create one will fail (Bad Request) in: query - schema: *ref_47 + schema: *ref_54 IncludeHeader: name: include_header description: > @@ -14664,14 +19125,14 @@ components: Header's key lowercased and '-'' replaced to '_' such that 'Content-Type' becomes the 'content_type' arg key in: query - schema: *ref_50 + schema: *ref_57 QueueLimit: name: queue_limit description: > The maximum size of the queue for which the request would get rejected if that job would push it above that limit in: query - schema: *ref_51 + schema: *ref_58 Payload: name: payload description: > @@ -14680,253 +19141,283 @@ components: `encodeURIComponent(btoa(JSON.stringify({a: 2})))` in: query - schema: *ref_94 + schema: *ref_113 ScriptStartPath: name: script_path_start description: mask to filter matching starting path in: query - schema: *ref_71 + schema: *ref_84 SchedulePath: name: schedule_path description: mask to filter by schedule path in: query - schema: *ref_72 + schema: *ref_85 ScriptExactPath: name: script_path_exact description: mask to filter exact matching path in: query - schema: *ref_70 + schema: *ref_83 ScriptExactHash: name: script_hash description: mask to filter exact matching path in: query - schema: *ref_73 + schema: *ref_86 CreatedBefore: name: created_before description: filter on created before (inclusive) timestamp in: query - schema: *ref_128 + schema: *ref_101 CreatedAfter: name: created_after description: filter on created after (exclusive) timestamp in: query - schema: *ref_129 + schema: *ref_102 StartedBefore: name: started_before description: filter on started before (inclusive) timestamp in: query - schema: *ref_74 + schema: *ref_87 StartedAfter: name: started_after description: filter on started after (exclusive) timestamp in: query - schema: *ref_75 + schema: *ref_88 Before: name: before description: filter on started before (inclusive) timestamp in: query - schema: *ref_120 + schema: *ref_183 CreatedOrStartedAfter: name: created_or_started_after description: >- filter on created_at for non non started job and started_at otherwise after (exclusive) timestamp in: query - schema: *ref_123 + schema: *ref_104 CreatedOrStartedAfterCompletedJob: name: created_or_started_after_completed_jobs description: >- filter on created_at for non non started job and started_at otherwise after (exclusive) timestamp but only for the completed jobs in: query - schema: *ref_124 + schema: *ref_105 CreatedOrStartedBefore: name: created_or_started_before description: >- filter on created_at for non non started job and started_at otherwise before (inclusive) timestamp in: query - schema: *ref_122 + schema: *ref_103 Success: name: success description: filter on successful jobs in: query - schema: *ref_76 + schema: *ref_96 ScheduledForBeforeNow: name: scheduled_for_before_now description: filter on jobs scheduled_for before now (hence waitinf for a worker) in: query - schema: *ref_77 + schema: *ref_90 Suspended: name: suspended description: filter on suspended jobs in: query - schema: *ref_79 + schema: *ref_92 Running: name: running description: filter on running jobs in: query - schema: *ref_80 + schema: *ref_89 + AllowWildcards: + name: allow_wildcards + description: allow wildcards (*) in the filter of label, tag, worker + in: query + schema: *ref_97 ArgsFilter: name: args description: filter on jobs containing those args as a json subset (@> in postgres) in: query - schema: *ref_81 + schema: *ref_93 Tag: name: tag description: filter on jobs with a given tag/worker group in: query - schema: *ref_83 + schema: *ref_94 ResultFilter: name: result description: filter on jobs containing those result as a json subset (@> in postgres) in: query - schema: *ref_82 + schema: *ref_95 After: name: after description: filter on created after (exclusive) timestamp in: query - schema: *ref_121 + schema: *ref_184 Username: name: username description: filter on exact username of user in: query - schema: *ref_130 + schema: *ref_189 Operation: name: operation description: filter on exact or prefix name of operation in: query - schema: *ref_131 + schema: *ref_190 ResourceName: name: resource description: filter on exact or prefix name of resource in: query - schema: *ref_132 + schema: *ref_191 ActionKind: name: action_kind description: filter on type of operation in: query - schema: *ref_133 + schema: *ref_192 JobKinds: name: job_kinds description: >- filter on job kind (values 'preview', 'script', 'dependencies', 'flow') separated by, in: query - schema: *ref_78 + schema: *ref_91 RunnableId: name: runnable_id in: query - schema: *ref_112 + schema: *ref_175 RunnableTypeQuery: name: runnable_type in: query - schema: *ref_113 + schema: *ref_176 InputId: name: input in: path required: true - schema: *ref_134 + schema: *ref_193 GetStarted: name: get_started in: query - schema: *ref_135 + schema: *ref_194 ConcurrencyId: name: concurrency_id in: path required: true - schema: *ref_136 + schema: *ref_195 + RunnableKind: + name: runnable_kind + in: path + required: true + schema: *ref_73 schemas: - AiResource: + InputTransform: + allOf: *ref_77 + AIProvider: + type: string + enum: *ref_196 + AIProviderModel: type: object - properties: *ref_16 - required: *ref_17 + properties: *ref_18 + required: *ref_19 + AIProviderConfig: + type: object + properties: *ref_197 + required: *ref_198 + AIConfig: + type: object + properties: *ref_20 + Alert: + type: object + properties: *ref_199 + required: *ref_200 + Configs: + type: object + nullable: true + properties: *ref_201 Script: - type: object - properties: *ref_37 - required: *ref_38 - NewScript: - type: object - properties: *ref_42 - required: *ref_43 - NewScriptWithDraft: - allOf: *ref_137 - ScriptHistory: type: object properties: *ref_44 required: *ref_45 + NewScript: + type: object + properties: *ref_49 + required: *ref_50 + NewScriptWithDraft: + allOf: *ref_202 + ScriptHistory: + type: object + properties: *ref_51 + required: *ref_52 ScriptArgs: type: object - additionalProperties: *ref_18 + additionalProperties: *ref_21 Input: type: object - properties: *ref_114 - required: *ref_115 + properties: *ref_177 + required: *ref_178 CreateInput: type: object - properties: *ref_138 - required: *ref_139 + properties: *ref_203 + required: *ref_204 UpdateInput: type: object - properties: *ref_140 - required: *ref_141 + properties: *ref_205 + required: *ref_206 RunnableType: type: string - enum: *ref_142 + enum: *ref_207 QueuedJob: type: object - properties: *ref_89 - required: *ref_90 + properties: *ref_108 + required: *ref_109 CompletedJob: type: object - properties: *ref_87 - required: *ref_88 + properties: *ref_106 + required: *ref_107 ObscuredJob: type: object - properties: *ref_143 + properties: *ref_208 Job: - oneOf: *ref_92 - discriminator: *ref_93 + oneOf: *ref_111 + discriminator: *ref_112 User: type: object properties: *ref_10 required: *ref_11 UserUsage: type: object - properties: *ref_144 + properties: *ref_209 Login: type: object - properties: *ref_145 - required: *ref_146 + properties: *ref_210 + required: *ref_211 EditWorkspaceUser: type: object - properties: *ref_147 + properties: *ref_212 TruncatedToken: type: object - properties: *ref_40 - required: *ref_41 + properties: *ref_47 + required: *ref_48 NewToken: type: object - properties: *ref_148 + properties: *ref_213 NewTokenImpersonate: type: object - properties: *ref_149 - required: *ref_150 + properties: *ref_214 + required: *ref_215 ListableVariable: type: object - properties: *ref_24 - required: *ref_25 + properties: *ref_27 + required: *ref_28 ContextualVariable: type: object - properties: *ref_151 - required: *ref_152 + properties: *ref_216 + required: *ref_217 CreateVariable: type: object - properties: *ref_153 - required: *ref_154 + properties: *ref_218 + required: *ref_219 EditVariable: type: object - properties: *ref_155 + properties: *ref_220 AuditLog: type: object properties: *ref_1 @@ -15056,164 +19547,315 @@ components: - error - no_main_func - has_preprocessor + ScriptLang: + type: string + enum: *ref_43 Preview: type: object - properties: *ref_156 - required: *ref_157 + properties: *ref_221 + required: *ref_222 WorkflowTask: + type: object + properties: *ref_223 + required: *ref_224 + WorkflowStatusRecord: + type: object + additionalProperties: *ref_225 + WorkflowStatus: + type: object + properties: *ref_226 + CreateResource: + type: object + properties: *ref_227 + required: *ref_228 + EditResource: + type: object + properties: *ref_229 + Resource: + type: object + properties: *ref_230 + required: *ref_231 + ListableResource: + type: object + properties: *ref_232 + required: *ref_233 + ResourceType: + type: object + properties: *ref_32 + required: *ref_33 + EditResourceType: + type: object + properties: *ref_234 + Schedule: + type: object + properties: *ref_115 + required: *ref_116 + ScheduleWJobs: + allOf: *ref_235 + NewSchedule: + type: object + properties: *ref_236 + required: *ref_237 + EditSchedule: + type: object + properties: *ref_238 + required: *ref_239 + TriggerExtraProperty: + type: object + properties: *ref_122 + required: *ref_123 + AuthenticationMethod: + type: string + enum: *ref_117 + HttpTrigger: + allOf: *ref_118 + type: object + properties: *ref_119 + required: *ref_120 + NewHttpTrigger: + type: object + properties: *ref_240 + required: *ref_241 + EditHttpTrigger: + type: object + properties: *ref_242 + required: *ref_243 + TriggersCount: + type: object + properties: *ref_66 + WebsocketTrigger: + allOf: *ref_124 + type: object + properties: *ref_125 + required: *ref_126 + NewWebsocketTrigger: + type: object + properties: *ref_244 + required: *ref_245 + EditWebsocketTrigger: + type: object + properties: *ref_246 + required: *ref_247 + WebsocketTriggerInitialMessage: + anyOf: *ref_121 + MqttQoS: + type: string + enum: *ref_248 + MqttV3Config: + type: object + properties: *ref_139 + MqttV5Config: + type: object + properties: *ref_140 + MqttSubscribeTopic: + type: object + properties: *ref_137 + required: *ref_138 + MqttClientVersion: + type: string + enum: *ref_141 + MqttTrigger: + allOf: *ref_142 + type: object + properties: *ref_143 + required: *ref_144 + NewMqttTrigger: + type: object + properties: *ref_249 + required: *ref_250 + EditMqttTrigger: + type: object + properties: *ref_251 + required: *ref_252 + DeliveryType: + type: string + enum: *ref_147 + PushConfig: + type: object + properties: *ref_148 + required: *ref_149 + GcpTrigger: + allOf: *ref_151 + type: object + properties: *ref_152 + required: *ref_153 + SubscriptionMode: + type: string + enum: *ref_150 + description: >- + The mode of subscription. 'existing' means using an existing GCP + subscription, while 'create_update' involves creating or updating a new + subscription. + GcpTriggerData: + type: object + properties: *ref_145 + required: *ref_146 + GetAllTopicSubscription: + type: object + properties: *ref_253 + required: *ref_254 + DeleteGcpSubscription: + type: object + properties: *ref_255 + required: *ref_256 + AwsAuthResourceType: + type: string + enum: *ref_133 + SqsTrigger: + allOf: *ref_134 + type: object + properties: *ref_135 + required: *ref_136 + NewSqsTrigger: + type: object + properties: *ref_257 + required: *ref_258 + EditSqsTrigger: + type: object + properties: *ref_259 + required: *ref_260 + Slot: + type: object + properties: *ref_154 + SlotList: + type: object + properties: *ref_261 + PublicationData: type: object properties: *ref_158 required: *ref_159 - WorkflowStatusRecord: + TableToTrack: + type: array + items: *ref_262 + Relations: type: object - additionalProperties: *ref_160 - WorkflowStatus: + properties: *ref_155 + required: *ref_156 + Language: + type: string + enum: *ref_263 + TemplateScript: + type: object + properties: *ref_264 + required: *ref_265 + PostgresTrigger: + allOf: *ref_160 type: object properties: *ref_161 - CreateResource: + required: *ref_162 + NewPostgresTrigger: type: object - properties: *ref_162 - required: *ref_163 - EditResource: + properties: *ref_266 + required: *ref_267 + EditPostgresTrigger: type: object - properties: *ref_164 - Resource: + properties: *ref_268 + required: *ref_269 + KafkaTrigger: + allOf: *ref_127 type: object - properties: *ref_165 - required: *ref_166 - ListableResource: + properties: *ref_128 + required: *ref_129 + NewKafkaTrigger: type: object - properties: *ref_167 - required: *ref_168 - ResourceType: + properties: *ref_270 + required: *ref_271 + EditKafkaTrigger: type: object - properties: *ref_28 - required: *ref_29 - EditResourceType: + properties: *ref_272 + required: *ref_273 + NatsTrigger: + allOf: *ref_130 type: object - properties: *ref_169 - Schedule: + properties: *ref_131 + required: *ref_132 + NewNatsTrigger: type: object - properties: *ref_96 - required: *ref_97 - ScheduleWJobs: - allOf: *ref_170 - NewSchedule: + properties: *ref_274 + required: *ref_275 + EditNatsTrigger: type: object - properties: *ref_171 - required: *ref_172 - EditSchedule: - type: object - properties: *ref_173 - required: *ref_174 - HttpTrigger: - type: object - properties: *ref_98 - required: *ref_99 - NewHttpTrigger: - type: object - properties: *ref_175 - required: *ref_176 - EditHttpTrigger: - type: object - properties: *ref_177 - required: *ref_178 - TriggersCount: - type: object - properties: *ref_59 - WebsocketTrigger: - type: object - properties: *ref_101 - required: *ref_102 - NewWebsocketTrigger: - type: object - properties: *ref_179 - required: *ref_180 - EditWebsocketTrigger: - type: object - properties: *ref_181 - required: *ref_182 - WebsocketTriggerInitialMessage: - anyOf: *ref_100 + properties: *ref_276 + required: *ref_277 Group: type: object - properties: *ref_108 - required: *ref_109 + properties: *ref_168 + required: *ref_169 InstanceGroup: type: object - properties: *ref_104 - required: *ref_105 + properties: *ref_164 + required: *ref_165 Folder: type: object - properties: *ref_110 - required: *ref_111 + properties: *ref_170 + required: *ref_171 WorkerPing: type: object - properties: *ref_183 - required: *ref_184 + properties: *ref_278 + required: *ref_279 UserWorkspaceList: type: object - properties: *ref_185 - required: *ref_186 + properties: *ref_280 + required: *ref_281 CreateWorkspace: type: object - properties: *ref_187 - required: *ref_188 + properties: *ref_282 + required: *ref_283 Workspace: type: object properties: *ref_7 required: *ref_8 WorkspaceInvite: type: object - properties: *ref_14 - required: *ref_15 + properties: *ref_16 + required: *ref_17 GlobalUserInfo: type: object - properties: *ref_12 - required: *ref_13 + properties: *ref_14 + required: *ref_15 Flow: - allOf: *ref_58 + allOf: *ref_65 ExtraPerms: type: object - additionalProperties: *ref_189 + additionalProperties: *ref_284 FlowMetadata: type: object - properties: *ref_190 - required: *ref_191 + properties: *ref_285 + required: *ref_286 OpenFlowWPath: - allOf: *ref_60 + allOf: *ref_67 FlowPreview: type: object - properties: *ref_192 - required: *ref_193 + properties: *ref_287 + required: *ref_288 RestartedFrom: type: object - properties: *ref_194 + properties: *ref_289 Policy: type: object - properties: *ref_61 + properties: *ref_68 ListableApp: type: object - properties: *ref_195 - required: *ref_196 + properties: *ref_290 + required: *ref_291 ListableRawApp: type: object - properties: *ref_197 - required: *ref_198 + properties: *ref_292 + required: *ref_293 AppWithLastVersion: type: object - properties: *ref_62 - required: *ref_63 + properties: *ref_69 + required: *ref_70 AppWithLastVersionWDraft: - allOf: *ref_199 + allOf: *ref_294 AppHistory: type: object - properties: *ref_64 - required: *ref_65 + properties: *ref_71 + required: *ref_72 FlowVersion: type: object - properties: *ref_56 - required: *ref_57 + properties: *ref_63 + required: *ref_64 SlackToken: type: object properties: @@ -15235,46 +19877,46 @@ components: - bot TokenResponse: type: object - properties: *ref_200 - required: *ref_201 + properties: *ref_295 + required: *ref_296 HubScriptKind: - name: kind - schema: *ref_36 + type: string + enum: *ref_42 PolarsClientKwargs: type: object - properties: *ref_118 - required: *ref_119 + properties: *ref_181 + required: *ref_182 LargeFileStorage: type: object - properties: *ref_19 + properties: *ref_22 WindmillLargeFile: type: object - properties: *ref_202 - required: *ref_203 + properties: *ref_297 + required: *ref_298 WindmillFileMetadata: type: object - properties: *ref_204 + properties: *ref_299 WindmillFilePreview: type: object - properties: *ref_205 - required: *ref_206 + properties: *ref_300 + required: *ref_301 S3Resource: type: object - properties: *ref_116 - required: *ref_117 + properties: *ref_179 + required: *ref_180 WorkspaceGitSyncSettings: type: object - properties: *ref_20 + properties: *ref_23 WorkspaceDeployUISettings: type: object - properties: *ref_21 + properties: *ref_24 WorkspaceDefaultScripts: type: object - properties: *ref_22 + properties: *ref_25 GitRepositorySettings: type: object - properties: *ref_207 - required: *ref_208 + properties: *ref_302 + required: *ref_303 UploadFilePart: type: object properties: @@ -15287,126 +19929,195 @@ components: - tag MetricMetadata: type: object - properties: *ref_209 - required: *ref_210 + properties: *ref_304 + required: *ref_305 ScalarMetric: type: object - properties: *ref_211 - required: *ref_212 + properties: *ref_306 + required: *ref_307 TimeseriesMetric: type: object - properties: *ref_213 - required: *ref_214 + properties: *ref_308 + required: *ref_309 MetricDataPoint: type: object - properties: *ref_215 - required: *ref_216 + properties: *ref_310 + required: *ref_311 RawScriptForDependencies: type: object - properties: *ref_217 - required: *ref_218 + properties: *ref_312 + required: *ref_313 ConcurrencyGroup: type: object - properties: *ref_219 - required: *ref_220 + properties: *ref_314 + required: *ref_315 ExtendedJobs: type: object - properties: *ref_221 - required: *ref_222 + properties: *ref_316 + required: *ref_317 ExportedUser: type: object properties: *ref_3 required: *ref_4 GlobalSetting: type: object - properties: *ref_223 - required: *ref_224 + properties: *ref_318 + required: *ref_319 Config: type: object - properties: *ref_225 - required: *ref_226 + properties: *ref_320 + required: *ref_321 ExportedInstanceGroup: type: object - properties: *ref_106 - required: *ref_107 + properties: *ref_166 + required: *ref_167 JobSearchHit: type: object - properties: *ref_227 + properties: *ref_322 LogSearchHit: type: object - properties: *ref_228 + properties: *ref_323 AutoscalingEvent: type: object - properties: *ref_229 + properties: *ref_324 CriticalAlert: type: object - properties: *ref_230 + properties: *ref_29 + CaptureTriggerKind: + type: string + enum: *ref_172 + Capture: + type: object + properties: *ref_173 + required: *ref_174 + CaptureConfig: + type: object + properties: *ref_325 + required: *ref_326 + OperatorSettings: + nullable: true + type: object + required: *ref_12 + properties: *ref_13 + TeamInfo: + type: object + required: *ref_327 + properties: *ref_328 + ChannelInfo: + type: object + required: *ref_329 + properties: *ref_330 + GithubInstallations: + type: array + items: *ref_331 + WorkspaceGithubInstallation: + type: object + properties: + account_id: + type: string + installation_id: + type: number + required: + - account_id + - installation_id + S3Object: + type: object + properties: *ref_75 + required: *ref_76 + TeamsChannel: + type: object + required: + - team_id + - team_name + - channel_id + - channel_name + properties: + team_id: + type: string + description: Microsoft Teams team ID + minLength: 1 + team_name: + type: string + description: Microsoft Teams team name + minLength: 1 + channel_id: + type: string + description: Microsoft Teams channel ID + minLength: 1 + channel_name: + type: string + description: Microsoft Teams channel name + minLength: 1 StaticTransform: type: object - properties: *ref_231 - required: *ref_232 + properties: *ref_332 + required: *ref_333 JavascriptTransform: type: object - properties: *ref_233 - required: *ref_234 - InputTransform: - oneOf: *ref_31 - discriminator: *ref_32 + properties: *ref_334 + required: *ref_335 + schemas-InputTransform: + oneOf: *ref_35 + discriminator: *ref_36 RawScript: type: object - properties: *ref_235 - required: *ref_236 + properties: *ref_336 + required: *ref_337 PathScript: type: object - properties: *ref_237 - required: *ref_238 + properties: *ref_338 + required: *ref_339 PathFlow: type: object - properties: *ref_239 - required: *ref_240 + properties: *ref_340 + required: *ref_341 FlowModule: type: object - properties: *ref_33 - required: *ref_34 + properties: *ref_37 + required: *ref_38 ForloopFlow: type: object - properties: *ref_241 - required: *ref_242 + properties: *ref_342 + required: *ref_343 WhileloopFlow: type: object - properties: *ref_243 - required: *ref_244 + properties: *ref_344 + required: *ref_345 BranchOne: type: object - properties: *ref_245 - required: *ref_246 + properties: *ref_346 + required: *ref_347 BranchAll: type: object - properties: *ref_247 - required: *ref_248 + properties: *ref_348 + required: *ref_349 Identity: type: object - properties: *ref_249 - required: *ref_250 + properties: *ref_350 + required: *ref_351 FlowModuleValue: - oneOf: *ref_251 - discriminator: *ref_252 + oneOf: *ref_352 + discriminator: *ref_353 + StopAfterIf: + type: object + properties: *ref_39 + required: *ref_40 Retry: type: object - properties: *ref_95 + properties: *ref_114 FlowValue: type: object - properties: *ref_66 - required: *ref_67 + properties: *ref_78 + required: *ref_79 OpenFlow: type: object - properties: *ref_54 - required: *ref_55 + properties: *ref_61 + required: *ref_62 FlowStatusModule: type: object - properties: *ref_68 - required: *ref_69 + properties: *ref_80 + required: *ref_81 FlowStatus: type: object - properties: *ref_84 - required: *ref_85 + properties: *ref_99 + required: *ref_100 diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index d1b0a39de5..049e8e3efb 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -1,7 +1,7 @@ openapi: "3.0.3" info: - version: 1.493.2 + version: 1.496.3 title: Windmill API contact: @@ -377,6 +377,9 @@ paths: type: string company: type: string + skip_email: + type: boolean + description: Skip sending email notifications to the user required: - email - password @@ -4498,6 +4501,13 @@ paths: in: query schema: type: boolean + - name: languages + in: query + description: | + Filter to only include scripts written in the given languages. + Accepts multiple values as a comma-separated list. + schema: + type: string responses: "200": description: All scripts @@ -4893,8 +4903,6 @@ paths: description: Script version/hash content: application/json: - required: false - schema: $ref: "#/components/schemas/ScriptHistory" @@ -5456,8 +5464,6 @@ paths: description: Flow version content: application/json: - required: false - schema: $ref: "#/components/schemas/FlowVersion" @@ -5487,8 +5493,7 @@ paths: operationId: getFlowVersion parameters: - $ref: "#/components/parameters/WorkspaceId" - - type: string - name: version + - name: version in: path required: true schema: @@ -5510,8 +5515,7 @@ paths: operationId: updateFlowHistory parameters: - $ref: "#/components/parameters/WorkspaceId" - - type: string - name: version + - name: version in: path required: true schema: @@ -6139,10 +6143,8 @@ paths: description: App version content: application/json: - required: false schema: $ref: "#/components/schemas/AppHistory" - /w/{workspace}/apps/list_paths_from_workspace_runnable/{runnable_kind}/{path}: get: summary: list app paths from workspace runnable @@ -6731,11 +6733,6 @@ paths: responses: "201": description: stream of created job uuids separated by \n. Lines may start with 'Error:' - example: | - a1a74c0d-708e-4539-9768-e8b3d37996bd - f0949132-5b30-48fe-bac8-873f047df810 - Error: Could not re-run 0b885808-ae89-4458-af95-c1ca3a13b0a5 - 52b9c01d-1125-4bbb-8bee-d41f26b70066 content: text/event-stream: schema: @@ -7592,7 +7589,8 @@ paths: description: job log content: text/plain: - type: string + schema: + type: string /w/{workspace}/jobs_u/get_flow_debug_info/{id}: get: @@ -8453,6 +8451,31 @@ paths: "201": description: default error handler set + /w/{workspace}/http_triggers/create_many: + post: + summary: create many HTTP triggers + operationId: createHttpTriggers + tags: + - http_trigger + parameters: + - $ref: "#/components/parameters/WorkspaceId" + requestBody: + description: new http trigger + required: true + content: + application/json: + schema: + type: array + items: + $ref: "#/components/schemas/NewHttpTrigger" + responses: + "201": + description: http trigger created + content: + text/plain: + schema: + type: string + /w/{workspace}/http_triggers/create: post: summary: create http trigger @@ -8604,8 +8627,7 @@ paths: route_path: type: string http_method: - type: string - enum: ["get", "post", "put", "delete", "patch"] + $ref: "#/components/schemas/HttpMethod" trigger_path: type: string workspaced_route: @@ -9830,6 +9852,23 @@ paths: items: type: string + /w/{workspace}/postgres_triggers/postgres/version/{path}: + get: + summary: get postgres version + operationId: getPostgresVersion + tags: + - postgres_trigger + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - $ref: "#/components/parameters/Path" + responses: + "200": + description: postgres version + content: + application/json: + schema: + type: string + /w/{workspace}/postgres_triggers/is_valid_postgres_configuration/{path}: get: summary: check if postgres configuration is set to logical @@ -12469,7 +12508,6 @@ paths: text/plain: schema: type: string - /concurrency_groups/list: get: summary: List all concurrency groups @@ -13175,7 +13213,51 @@ components: enum: [script, flow] schemas: - $ref: "../../openflow.openapi.yaml#/components/schemas" + # NOTE: Not so many generators and validators support this format: + # $ref: "../../openflow.openapi.yaml#/components/schemas" + # This is why it is better to inline each of schemas for better compat + # Do not change next line. It is used by python-client for pre-processing + # -- INLINE START -- + OpenFlow: + $ref: "../../openflow.openapi.yaml#/components/schemas/OpenFlow" + FlowValue: + $ref: "../../openflow.openapi.yaml#/components/schemas/FlowValue" + Retry: + $ref: "../../openflow.openapi.yaml#/components/schemas/Retry" + StopAfterIf: + $ref: "../../openflow.openapi.yaml#/components/schemas/StopAfterIf" + FlowModule: + $ref: "../../openflow.openapi.yaml#/components/schemas/FlowModule" + InputTransform: + $ref: "../../openflow.openapi.yaml#/components/schemas/InputTransform" + StaticTransform: + $ref: "../../openflow.openapi.yaml#/components/schemas/StaticTransform" + JavascriptTransform: + $ref: "../../openflow.openapi.yaml#/components/schemas/JavascriptTransform" + FlowModuleValue: + $ref: "../../openflow.openapi.yaml#/components/schemas/FlowModuleValue" + RawScript: + $ref: "../../openflow.openapi.yaml#/components/schemas/RawScript" + PathScript: + $ref: "../../openflow.openapi.yaml#/components/schemas/PathScript" + PathFlow: + $ref: "../../openflow.openapi.yaml#/components/schemas/PathFlow" + ForloopFlow: + $ref: "../../openflow.openapi.yaml#/components/schemas/ForloopFlow" + WhileloopFlow: + $ref: "../../openflow.openapi.yaml#/components/schemas/WhileloopFlow" + BranchOne: + $ref: "../../openflow.openapi.yaml#/components/schemas/BranchOne" + BranchAll: + $ref: "../../openflow.openapi.yaml#/components/schemas/BranchAll" + Identity: + $ref: "../../openflow.openapi.yaml#/components/schemas/Identity" + FlowStatus: + $ref: "../../openflow.openapi.yaml#/components/schemas/FlowStatus" + FlowStatusModule: + $ref: "../../openflow.openapi.yaml#/components/schemas/FlowStatusModule" + # -- INLINE END -- + # Do not change line above AIProvider: type: string @@ -14672,6 +14754,15 @@ components: - custom_script - signature + HttpMethod: + type: string + enum: + - get + - post + - put + - delete + - patch + HttpTrigger: allOf: - $ref: "#/components/schemas/TriggerExtraProperty" @@ -14691,13 +14782,7 @@ components: required: - s3 http_method: - type: string - enum: - - get - - post - - put - - delete - - patch + $ref: "#/components/schemas/HttpMethod" authentication_resource_path: type: string is_async: @@ -14748,13 +14833,7 @@ components: is_flow: type: boolean http_method: - type: string - enum: - - get - - post - - put - - delete - - patch + $ref: "#/components/schemas/HttpMethod" authentication_resource_path: type: string is_async: @@ -14805,13 +14884,7 @@ components: is_flow: type: boolean http_method: - type: string - enum: - - get - - post - - put - - delete - - patch + $ref: "#/components/schemas/HttpMethod" is_async: type: boolean authentication_method: @@ -16225,10 +16298,8 @@ components: - access_token HubScriptKind: - name: kind - schema: - type: string - enum: [script, failure, trigger, approval] + type: string + enum: [script, failure, trigger, approval] PolarsClientKwargs: type: object diff --git a/backend/windmill-api/src/agent_workers_ee.rs b/backend/windmill-api/src/agent_workers_oss.rs similarity index 76% rename from backend/windmill-api/src/agent_workers_ee.rs rename to backend/windmill-api/src/agent_workers_oss.rs index 1b5a210d13..655058c493 100644 --- a/backend/windmill-api/src/agent_workers_ee.rs +++ b/backend/windmill-api/src/agent_workers_oss.rs @@ -1,3 +1,7 @@ +#[cfg(feature = "private")] +#[allow(unused)] +pub use crate::agent_workers_ee::*; + /* * Author: Ruben Fiszel * Copyright: Windmill Labs, Inc 2042 @@ -6,16 +10,21 @@ * LICENSE-AGPL for a copy of the license. */ +#[cfg(not(feature = "private"))] use crate::db::DB; +#[cfg(not(feature = "private"))] use axum::Router; +#[cfg(not(feature = "private"))] use serde::{Deserialize, Serialize}; +#[cfg(not(feature = "private"))] pub fn global_service() -> Router { Router::new() } +#[cfg(not(feature = "private"))] pub fn workspaced_service( db: DB, _base_internal_url: String, @@ -36,6 +45,7 @@ pub fn workspaced_service( } #[derive(Clone, Debug, Deserialize, Serialize)] +#[cfg(not(feature = "private"))] pub struct AgentAuth { pub worker_group: String, pub suffix: Option, @@ -43,8 +53,10 @@ pub struct AgentAuth { pub exp: Option, } +#[cfg(not(feature = "private"))] pub struct AgentCache {} +#[cfg(not(feature = "private"))] impl AgentCache { pub fn new() -> Self { AgentCache {} diff --git a/backend/windmill-api/src/ai.rs b/backend/windmill-api/src/ai.rs index 6b33a70730..86e69fe233 100644 --- a/backend/windmill-api/src/ai.rs +++ b/backend/windmill-api/src/ai.rs @@ -10,7 +10,7 @@ use reqwest::{Client, RequestBuilder}; use serde::{Deserialize, Serialize}; use serde_json::value::RawValue; use std::collections::HashMap; -use windmill_audit::{audit_ee::audit_log, ActionKind}; +use windmill_audit::{audit_oss::audit_log, ActionKind}; use windmill_common::error::{to_anyhow, Error, Result}; lazy_static::lazy_static! { diff --git a/backend/windmill-api/src/apps.rs b/backend/windmill-api/src/apps.rs index 533da11d1c..9dfe40b5b9 100644 --- a/backend/windmill-api/src/apps.rs +++ b/backend/windmill-api/src/apps.rs @@ -18,7 +18,7 @@ use crate::{ }; #[cfg(feature = "parquet")] use crate::{ - job_helpers_ee::{ + job_helpers_oss::{ download_s3_file_internal, get_random_file_name, get_s3_resource, get_workspace_s3_resource, upload_file_from_req, DownloadFileQuery, }, @@ -48,7 +48,7 @@ use sha2::{Digest, Sha256}; use sql_builder::{bind::Bind, SqlBuilder}; use sqlx::{types::Uuid, FromRow}; use std::str; -use windmill_audit::audit_ee::audit_log; +use windmill_audit::audit_oss::audit_log; use windmill_audit::ActionKind; use windmill_common::{ apps::{AppScriptId, ListAppQuery}, diff --git a/backend/windmill-api/src/apps_ee.rs b/backend/windmill-api/src/apps_ee.rs deleted file mode 100644 index a7737664b9..0000000000 --- a/backend/windmill-api/src/apps_ee.rs +++ /dev/null @@ -1,5 +0,0 @@ -use axum::Router; - -pub fn global_unauthed_service() -> Router { - Router::new() -} diff --git a/backend/windmill-api/src/apps_oss.rs b/backend/windmill-api/src/apps_oss.rs new file mode 100644 index 0000000000..346fff9ff1 --- /dev/null +++ b/backend/windmill-api/src/apps_oss.rs @@ -0,0 +1,11 @@ +#[cfg(feature = "private")] +#[allow(unused)] +pub use crate::apps_ee::*; + +#[cfg(not(feature = "private"))] +use axum::Router; + +#[cfg(not(feature = "private"))] +pub fn global_unauthed_service() -> Router { + Router::new() +} diff --git a/backend/windmill-api/src/args.rs b/backend/windmill-api/src/args.rs index e3a2adb38f..0b8f4d5210 100644 --- a/backend/windmill-api/src/args.rs +++ b/backend/windmill-api/src/args.rs @@ -85,7 +85,7 @@ impl RawWebhookArgs { db: &DB, w_id: &str, ) -> Result>, Error> { - use crate::job_helpers_ee::{ + use crate::job_helpers_oss::{ get_random_file_name, get_workspace_s3_resource, upload_file_internal, }; use futures::TryStreamExt; diff --git a/backend/windmill-api/src/audit.rs b/backend/windmill-api/src/audit.rs index 47ba881014..336fd32881 100644 --- a/backend/windmill-api/src/audit.rs +++ b/backend/windmill-api/src/audit.rs @@ -28,7 +28,7 @@ async fn get_audit( Path((w_id, id)): Path<(String, i32)>, ) -> JsonResult { let tx = user_db.begin(&authed).await?; - let audit = windmill_audit::audit_ee::get_audit(tx, id, &w_id).await?; + let audit = windmill_audit::audit_oss::get_audit(tx, id, &w_id).await?; Ok(Json(audit)) } async fn list_audit( @@ -39,6 +39,6 @@ async fn list_audit( Query(lq): Query, ) -> JsonResult> { let tx = user_db.begin(&authed).await?; - let rows = windmill_audit::audit_ee::list_audit(tx, w_id, pagination, lq).await?; + let rows = windmill_audit::audit_oss::list_audit(tx, w_id, pagination, lq).await?; Ok(Json(rows)) } diff --git a/backend/windmill-api/src/auth.rs b/backend/windmill-api/src/auth.rs index 2936b74957..6af5ea1e50 100644 --- a/backend/windmill-api/src/auth.rs +++ b/backend/windmill-api/src/auth.rs @@ -1,5 +1,5 @@ #[cfg(feature = "enterprise")] -use crate::ee::ExternalJwks; +use crate::ee_oss::ExternalJwks; use axum::{ async_trait, extract::{FromRequestParts, OriginalUri, Query}, @@ -26,6 +26,20 @@ use windmill_common::{ users::{COOKIE_NAME, SUPERADMIN_SECRET_EMAIL}, }; +lazy_static::lazy_static! { + // Global auth cache accessible from main.rs for direct invalidation + pub static ref AUTH_CACHE: Cache<(String, String), ExpiringAuthCache> = Cache::new(300); +} + +// Global function to invalidate a specific token from cache +pub fn invalidate_token_from_cache(token: &str) { + // Remove all cache entries for this token (across all workspaces) + AUTH_CACHE.retain(|(_workspace_id, cached_token), _cached_value| { + cached_token != token + }); + tracing::info!("Invalidated token from auth cache: {}...", &token[..token.len().min(8)]); +} + #[derive(Clone)] pub struct ExpiringAuthCache { pub authed: ApiAuthed, @@ -33,7 +47,6 @@ pub struct ExpiringAuthCache { } pub struct AuthCache { - cache: Cache<(String, String), ExpiringAuthCache>, db: DB, superadmin_secret: Option, #[cfg(feature = "enterprise")] @@ -47,7 +60,6 @@ impl AuthCache { #[cfg(feature = "enterprise")] ext_jwks: Option>>, ) -> Self { AuthCache { - cache: Cache::new(300), db, superadmin_secret, #[cfg(feature = "enterprise")] @@ -56,7 +68,7 @@ impl AuthCache { } pub async fn invalidate(&self, w_id: &str, token: String) { - self.cache.remove(&(w_id.to_string(), token)); + AUTH_CACHE.remove(&(w_id.to_string(), token)); } pub async fn get_authed(&self, w_id: Option, token: &str) -> Option { @@ -64,14 +76,14 @@ impl AuthCache { w_id.as_ref().unwrap_or(&"".to_string()).to_string(), token.to_string(), ); - let s = self.cache.get(&key).map(|c| c.to_owned()); + let s = AUTH_CACHE.get(&key).map(|c| c.to_owned()); match s { Some(ExpiringAuthCache { authed, expiry }) if expiry > chrono::Utc::now() => { Some(authed) } #[cfg(feature = "enterprise")] _ if token.starts_with("jwt_ext_") => { - let authed_and_exp = match crate::ee::jwt_ext_auth( + let authed_and_exp = match crate::ee_oss::jwt_ext_auth( w_id.as_ref(), token.trim_start_matches("jwt_ext_"), self.ext_jwks.clone(), @@ -86,7 +98,7 @@ impl AuthCache { }; if let Some((authed, exp)) = authed_and_exp.clone() { - self.cache.insert( + AUTH_CACHE.insert( key, ExpiringAuthCache { authed: authed.clone(), @@ -123,7 +135,7 @@ impl AuthCache { username_override, }; - self.cache.insert( + AUTH_CACHE.insert( key, ExpiringAuthCache { authed: authed.clone(), @@ -317,7 +329,7 @@ impl AuthCache { } }; if let Some(authed) = authed_o.as_ref() { - self.cache.insert( + AUTH_CACHE.insert( key, ExpiringAuthCache { authed: authed.clone(), diff --git a/backend/windmill-api/src/capture.rs b/backend/windmill-api/src/capture.rs index 07127e53a1..0c2d9f3946 100644 --- a/backend/windmill-api/src/capture.rs +++ b/backend/windmill-api/src/capture.rs @@ -15,7 +15,7 @@ use { #[cfg(all(feature = "enterprise", feature = "gcp_trigger"))] use { - crate::gcp_triggers_ee::{ + crate::gcp_triggers_oss::{ manage_google_subscription, process_google_push_request, validate_jwt_token, CreateUpdateConfig, SubscriptionMode, }, @@ -23,7 +23,10 @@ use { http::HeaderMap, }; -#[cfg(all(feature = "enterprise", feature = "gcp_trigger"))] +#[cfg(any( + all(feature = "enterprise", feature = "gcp_trigger"), + feature = "postgres_trigger" +))] use windmill_common::utils::empty_as_none; #[cfg(all(feature = "enterprise", feature = "sqs_trigger"))] @@ -33,25 +36,28 @@ use windmill_common::auth::aws::AwsAuthResourceType; feature = "http_trigger", all(feature = "enterprise", feature = "gcp_trigger") ))] -use {serde::de::DeserializeOwned, windmill_common::error::Error}; +use serde::de::DeserializeOwned; + +#[cfg(any( + feature = "http_trigger", + feature = "postgres_trigger", + all(feature = "enterprise", feature = "gcp_trigger") +))] +use windmill_common::error::Error; #[cfg(all(feature = "enterprise", feature = "kafka"))] -use crate::kafka_triggers_ee::KafkaTriggerConfigConnection; +use crate::kafka_triggers_oss::KafkaTriggerConfigConnection; #[cfg(feature = "mqtt_trigger")] use crate::mqtt_triggers::{MqttClientVersion, MqttV3Config, MqttV5Config, SubscribeTopic}; #[cfg(all(feature = "enterprise", feature = "nats"))] -use crate::nats_triggers_ee::NatsTriggerConfigConnection; +use crate::nats_triggers_oss::NatsTriggerConfigConnection; #[cfg(feature = "postgres_trigger")] -use { - crate::postgres_triggers::{ - create_logical_replication_slot_query, create_publication_query, drop_publication_query, - generate_random_string, get_database_connection, PublicationData, - }, - itertools::Itertools, - pg_escape::quote_literal, +use crate::postgres_triggers::{ + create_logical_replication_slot, create_pg_publication, generate_random_string, + get_default_pg_connection, PublicationData, }; use crate::{ @@ -200,9 +206,12 @@ pub struct MqttTriggerConfig { #[derive(Serialize, Deserialize, Debug)] pub struct PostgresTriggerConfig { pub postgres_resource_path: String, + #[serde(default, deserialize_with = "empty_as_none")] pub publication_name: Option, + #[serde(default, deserialize_with = "empty_as_none")] pub replication_slot_name: Option, pub publication: PublicationData, + pub basic_mode: Option, } #[cfg(feature = "websocket")] @@ -292,57 +301,55 @@ async fn set_postgres_trigger_config( user_db: UserDB, mut capture_config: NewCaptureConfig, ) -> Result { - let Some(TriggerConfig::Postgres(mut postgres_config)) = capture_config.trigger_config else { - return Err(windmill_common::error::Error::BadRequest( - "Invalid postgres config".to_string(), - )); + use windmill_common::error::to_anyhow; + + let Some(TriggerConfig::Postgres(postgres_config)) = capture_config.trigger_config.as_mut() + else { + return Err(Error::BadRequest("Invalid postgres config".to_string())); }; - let mut connection = get_database_connection( - authed, - Some(user_db), - &db, - &postgres_config.postgres_resource_path, - &w_id, - ) - .await?; + if postgres_config.basic_mode.unwrap_or(false) { + let mut pg_connection = get_default_pg_connection( + authed, + Some(user_db), + &db, + &postgres_config.postgres_resource_path, + &w_id, + ) + .await?; - let publication_name = postgres_config - .publication_name - .get_or_insert(format!("windmill_capture_{}", generate_random_string())); - let replication_slot_name = postgres_config - .replication_slot_name - .get_or_insert(publication_name.clone()); + let tx = pg_connection.transaction().await.map_err(to_anyhow)?; - let query = drop_publication_query(&publication_name); + let publication_name = format!("windmill_capture_{}", generate_random_string()); + let replication_slot_name = publication_name.clone(); - sqlx::query(&query).execute(&mut connection).await?; + create_logical_replication_slot(tx.client(), &replication_slot_name) + .await + .map_err(to_anyhow)?; - let query = create_publication_query( - &publication_name, - postgres_config.publication.table_to_track.as_deref(), - &postgres_config - .publication - .transaction_to_track - .iter() - .map(AsRef::as_ref) - .collect_vec(), - ); + create_pg_publication( + tx.client(), + &publication_name, + postgres_config.publication.table_to_track.as_deref(), + &postgres_config.publication.transaction_to_track, + ) + .await + .map_err(to_anyhow)?; - sqlx::query(&query).execute(&mut connection).await?; + tx.commit().await.map_err(to_anyhow)?; - let query = format!( - "SELECT 1 from pg_replication_slots WHERE slot_name = {}", - quote_literal(replication_slot_name) - ); - - let row = sqlx::query(&query).fetch_optional(&mut connection).await?; - - if row.is_none() { - let query = create_logical_replication_slot_query(&replication_slot_name); - sqlx::query(&query).execute(&mut connection).await?; + postgres_config.publication_name = Some(publication_name); + postgres_config.replication_slot_name = Some(replication_slot_name); + } else { + if postgres_config.publication_name.is_none() + || postgres_config.replication_slot_name.is_none() + { + return Err(Error::BadRequest( + "Publication name and slot name required in advanced mode".to_string(), + )); + } } - capture_config.trigger_config = Some(TriggerConfig::Postgres(postgres_config)); + Ok(capture_config) } @@ -366,9 +373,7 @@ async fn set_gcp_trigger_config( mut capture_config: NewCaptureConfig, ) -> Result { let Some(TriggerConfig::Gcp(mut gcp_config)) = capture_config.trigger_config else { - return Err(windmill_common::error::Error::BadRequest( - "Invalid GCP Pub/Sub config".to_string(), - )); + return Err(Error::BadRequest("Invalid GCP Pub/Sub config".to_string())); }; let config = manage_google_subscription( @@ -903,7 +908,7 @@ async fn gcp_payload( headers: HeaderMap, request: Request, ) -> Result { - use crate::{gcp_triggers_ee::GcpTrigger, trigger_helpers::TriggerJobArgs}; + use crate::{gcp_triggers_oss::GcpTrigger, trigger_helpers::TriggerJobArgs}; let is_flow = matches!(runnable_kind, RunnableKind::Flow); let (gcp_trigger_config, owner, email): (GcpTriggerConfig, _, _) = diff --git a/backend/windmill-api/src/configs.rs b/backend/windmill-api/src/configs.rs index e8770b2b5e..9110d104c8 100644 --- a/backend/windmill-api/src/configs.rs +++ b/backend/windmill-api/src/configs.rs @@ -14,7 +14,7 @@ use axum::{ use serde::{Deserialize, Serialize}; use sqlx::FromRow; -use windmill_audit::audit_ee::audit_log; +use windmill_audit::audit_oss::audit_log; use windmill_audit::ActionKind; use windmill_common::{ error::{self}, diff --git a/backend/windmill-api/src/db.rs b/backend/windmill-api/src/db.rs index 85f1ee485f..c8f76dd817 100644 --- a/backend/windmill-api/src/db.rs +++ b/backend/windmill-api/src/db.rs @@ -16,7 +16,7 @@ use sqlx::{ }; use tokio::task::JoinHandle; -use windmill_audit::audit_ee::{AuditAuthor, AuditAuthorable}; +use windmill_audit::audit_oss::{AuditAuthor, AuditAuthorable}; use windmill_common::{ db::{Authable, Authed}, error::Error, @@ -804,6 +804,14 @@ async fn fix_job_completed_index(db: &DB) -> Result<(), Error> { .execute(db) .await?; }); + + run_windmill_migration!("v2_job_queue_suspend", db, |tx| { + sqlx::query!( + "CREATE INDEX CONCURRENTLY IF NOT EXISTS v2_job_queue_suspend ON v2_job_queue (workspace_id, suspend) WHERE suspend > 0;" + ) + .execute(db) + .await?; + }); Ok(()) } diff --git a/backend/windmill-api/src/ee.rs b/backend/windmill-api/src/ee_oss.rs similarity index 61% rename from backend/windmill-api/src/ee.rs rename to backend/windmill-api/src/ee_oss.rs index cddb639e95..3fb1271a73 100644 --- a/backend/windmill-api/src/ee.rs +++ b/backend/windmill-api/src/ee_oss.rs @@ -1,15 +1,21 @@ +#[cfg(feature = "private")] +#[allow(unused)] +pub use crate::ee::*; + +#[cfg(not(feature = "private"))] use anyhow::anyhow; -#[cfg(feature = "enterprise")] +#[cfg(all(feature = "enterprise", not(feature = "private")))] use std::sync::Arc; -#[cfg(feature = "enterprise")] +#[cfg(all(feature = "enterprise", not(feature = "private")))] use tokio::sync::RwLock; +#[cfg(not(feature = "private"))] pub async fn validate_license_key(_license_key: String) -> anyhow::Result<(String, bool)> { // Implementation is not open source Err(anyhow!("License can't be validated in Windmill CE")) } -#[cfg(feature = "enterprise")] +#[cfg(all(feature = "enterprise", not(feature = "private")))] pub async fn jwt_ext_auth( _w_id: Option<&String>, _token: &str, @@ -20,10 +26,10 @@ pub async fn jwt_ext_auth( Err(anyhow!("External JWT auth is not open source")) } -#[cfg(feature = "enterprise")] +#[cfg(all(feature = "enterprise", not(feature = "private")))] pub struct ExternalJwks; -#[cfg(feature = "enterprise")] +#[cfg(all(feature = "enterprise", not(feature = "private")))] impl ExternalJwks { pub async fn load() -> Option>> { // Implementation is not open source diff --git a/backend/windmill-api/src/flows.rs b/backend/windmill-api/src/flows.rs index e34b2cacaf..505656da47 100644 --- a/backend/windmill-api/src/flows.rs +++ b/backend/windmill-api/src/flows.rs @@ -31,7 +31,7 @@ use hyper::StatusCode; use serde::{Deserialize, Serialize}; use sql_builder::prelude::*; use sqlx::{FromRow, Postgres, Transaction}; -use windmill_audit::audit_ee::audit_log; +use windmill_audit::audit_oss::audit_log; use windmill_audit::ActionKind; use windmill_common::utils::query_elems_from_hub; use windmill_common::worker::to_raw_value; diff --git a/backend/windmill-api/src/folders.rs b/backend/windmill-api/src/folders.rs index 5047a96cad..1e4d2e09c0 100644 --- a/backend/windmill-api/src/folders.rs +++ b/backend/windmill-api/src/folders.rs @@ -23,7 +23,7 @@ use axum::{ }; use lazy_static::lazy_static; use regex::Regex; -use windmill_audit::audit_ee::audit_log; +use windmill_audit::audit_oss::audit_log; use windmill_audit::ActionKind; use windmill_common::{ db::UserDB, diff --git a/backend/windmill-api/src/gcp_triggers_ee.rs b/backend/windmill-api/src/gcp_triggers_oss.rs similarity index 76% rename from backend/windmill-api/src/gcp_triggers_ee.rs rename to backend/windmill-api/src/gcp_triggers_oss.rs index 0dc672580c..78005309bc 100644 --- a/backend/windmill-api/src/gcp_triggers_ee.rs +++ b/backend/windmill-api/src/gcp_triggers_oss.rs @@ -1,29 +1,38 @@ -use crate::db::{ApiAuthed, DB}; -use crate::trigger_helpers::TriggerJobArgs; -use axum::{extract::Request, Router}; -use http::HeaderMap; -use serde::{Deserialize, Serialize}; -use serde_json::value::RawValue; -use sqlx::prelude::FromRow; -use sqlx::types::Json as SqlxJson; -use std::collections::HashMap; -use windmill_common::db::UserDB; -use windmill_common::worker::to_raw_value; -use windmill_common::{ - error::{Error as WindmillError, Result as WindmillResult}, - triggers::TriggerKind, - utils::empty_as_none, +#[cfg(feature = "private")] +#[allow(unused)] +pub use crate::gcp_triggers_ee::*; + +#[cfg(not(feature = "private"))] +use { + crate::db::{ApiAuthed, DB}, + crate::trigger_helpers::TriggerJobArgs, + axum::{extract::Request, Router}, + http::HeaderMap, + serde::{Deserialize, Serialize}, + serde_json::value::RawValue, + sqlx::prelude::FromRow, + sqlx::types::Json as SqlxJson, + std::collections::HashMap, + windmill_common::db::UserDB, + windmill_common::worker::to_raw_value, + windmill_common::{ + error::{Error as WindmillError, Result as WindmillResult}, + triggers::TriggerKind, + utils::empty_as_none, + }, }; #[derive(sqlx::Type, Debug, Deserialize, Serialize)] #[serde(rename_all(serialize = "lowercase", deserialize = "lowercase"))] #[sqlx(type_name = "DELIVERY_MODE", rename_all = "lowercase")] #[allow(unused)] +#[cfg(not(feature = "private"))] pub enum DeliveryType { Pull, Push, } +#[cfg(not(feature = "private"))] impl Default for DeliveryType { fn default() -> Self { Self::Pull @@ -32,6 +41,7 @@ impl Default for DeliveryType { #[derive(FromRow, Deserialize, Serialize, Debug)] #[allow(unused)] +#[cfg(not(feature = "private"))] pub struct PushConfig { #[serde(deserialize_with = "empty_as_none")] route_path: Option, @@ -42,6 +52,7 @@ pub struct PushConfig { } #[derive(Default, Debug, Serialize, Deserialize)] #[allow(unused)] +#[cfg(not(feature = "private"))] pub struct CreateUpdateConfig { pub delivery_type: DeliveryType, #[serde(default, deserialize_with = "empty_as_none")] @@ -50,6 +61,7 @@ pub struct CreateUpdateConfig { } #[derive(Debug, Deserialize, Serialize)] +#[cfg(not(feature = "private"))] pub struct ExistingGcpSubscription { pub subscription_id: String, pub base_endpoint: String, @@ -58,15 +70,18 @@ pub struct ExistingGcpSubscription { #[derive(Debug, Deserialize, Serialize, sqlx::Type)] #[serde(rename_all = "snake_case")] #[sqlx(type_name = "GCP_SUBSCRIPTION_MODE", rename_all = "snake_case")] +#[cfg(not(feature = "private"))] pub enum SubscriptionMode { Existing, CreateUpdate, } +#[cfg(not(feature = "private"))] pub fn workspaced_service() -> Router { Router::new() } +#[cfg(not(feature = "private"))] pub fn start_consuming_gcp_pubsub_event( _db: DB, mut _killpill_rx: tokio::sync::broadcast::Receiver<()>, @@ -74,6 +89,7 @@ pub fn start_consuming_gcp_pubsub_event( // implementation is not open source } +#[cfg(not(feature = "private"))] pub async fn manage_google_subscription( _authed: ApiAuthed, _db: &DB, @@ -91,6 +107,7 @@ pub async fn manage_google_subscription( Ok(CreateUpdateConfig::default()) } +#[cfg(not(feature = "private"))] pub async fn process_google_push_request( _headers: HeaderMap, _request: Request, @@ -98,6 +115,7 @@ pub async fn process_google_push_request( Ok((String::new(), HashMap::new())) } +#[cfg(not(feature = "private"))] pub async fn validate_jwt_token( _db: &DB, _user_db: UserDB, @@ -110,11 +128,13 @@ pub async fn validate_jwt_token( Ok(()) } +#[cfg(not(feature = "private"))] pub fn gcp_push_route_handler() -> Router { Router::new() } #[derive(FromRow, Deserialize, Serialize, Debug)] +#[cfg(not(feature = "private"))] pub struct GcpTrigger { pub gcp_resource_path: String, pub subscription_id: String, @@ -135,7 +155,7 @@ pub struct GcpTrigger { pub last_server_ping: Option>, pub enabled: bool, } - +#[cfg(not(feature = "private"))] impl TriggerJobArgs for GcpTrigger { fn v1_payload_fn(payload: String) -> HashMap> { HashMap::from([("payload".to_string(), to_raw_value(&payload))]) diff --git a/backend/windmill-api/src/git_sync_ee.rs b/backend/windmill-api/src/git_sync_ee.rs deleted file mode 100644 index b72afac189..0000000000 --- a/backend/windmill-api/src/git_sync_ee.rs +++ /dev/null @@ -1,9 +0,0 @@ -use axum::routing::Router; - -pub fn workspaced_service() -> Router { - Router::new() -} - -pub fn global_service() -> Router { - Router::new() -} \ No newline at end of file diff --git a/backend/windmill-api/src/git_sync_oss.rs b/backend/windmill-api/src/git_sync_oss.rs new file mode 100644 index 0000000000..0451d88699 --- /dev/null +++ b/backend/windmill-api/src/git_sync_oss.rs @@ -0,0 +1,16 @@ +#[cfg(feature = "private")] +#[allow(unused)] +pub use crate::git_sync_ee::*; + +#[cfg(not(feature = "private"))] +use axum::routing::Router; + +#[cfg(not(feature = "private"))] +pub fn workspaced_service() -> Router { + Router::new() +} + +#[cfg(not(feature = "private"))] +pub fn global_service() -> Router { + Router::new() +} diff --git a/backend/windmill-api/src/groups.rs b/backend/windmill-api/src/groups.rs index e25d3ee7da..d5da9c25f5 100644 --- a/backend/windmill-api/src/groups.rs +++ b/backend/windmill-api/src/groups.rs @@ -14,7 +14,7 @@ use axum::{ routing::{delete, get, post}, Json, Router, }; -use windmill_audit::audit_ee::audit_log; +use windmill_audit::audit_oss::audit_log; use windmill_audit::ActionKind; use windmill_common::worker::CLOUD_HOSTED; use windmill_common::{ diff --git a/backend/windmill-api/src/http_trigger_args.rs b/backend/windmill-api/src/http_trigger_args.rs index 6aab75e46e..8fb96d42f0 100644 --- a/backend/windmill-api/src/http_trigger_args.rs +++ b/backend/windmill-api/src/http_trigger_args.rs @@ -21,7 +21,7 @@ use crate::{ pub struct RawHttpTriggerArgs(pub RawWebhookArgs); -#[derive(Serialize, Deserialize, sqlx::Type, Debug, Clone, Hash, Eq, PartialEq)] +#[derive(Serialize, Deserialize, sqlx::Type, Debug, Clone, Copy, Hash, Eq, PartialEq)] #[sqlx(type_name = "HTTP_METHOD", rename_all = "lowercase")] #[serde(rename_all = "lowercase")] pub enum HttpMethod { diff --git a/backend/windmill-api/src/http_triggers.rs b/backend/windmill-api/src/http_triggers.rs index b17c8205ad..22a534d1ef 100644 --- a/backend/windmill-api/src/http_triggers.rs +++ b/backend/windmill-api/src/http_triggers.rs @@ -1,7 +1,7 @@ #[cfg(feature = "http_trigger")] use crate::http_trigger_args::{HttpMethod, RawHttpTriggerArgs}; #[cfg(feature = "parquet")] -use crate::job_helpers_ee::get_workspace_s3_resource; +use crate::job_helpers_oss::get_workspace_s3_resource; use crate::resources::try_get_resource_from_db_as; use crate::trigger_helpers::{get_runnable_format, RunnableId}; use crate::utils::{non_empty_str, ExpiringCacheEntry}; @@ -14,6 +14,7 @@ use crate::{ }, users::fetch_api_authed, }; +use anyhow::anyhow; use axum::response::Response; use axum::{ extract::{Path, Query}, @@ -28,13 +29,14 @@ use quick_cache::sync::Cache; use serde::{Deserialize, Serialize}; use sql_builder::{bind::Bind, SqlBuilder}; use sqlx::prelude::FromRow; -use sqlx::PgTransaction; +use sqlx::PgConnection; use std::borrow::Cow; +use std::collections::HashSet; use std::{collections::HashMap, sync::Arc}; use tokio::sync::{RwLock, RwLockReadGuard}; use tower_http::cors::CorsLayer; -use windmill_audit::{audit_ee::audit_log, ActionKind}; -use windmill_common::error::Error; +use windmill_audit::{audit_oss::audit_log, ActionKind}; +use windmill_common::error::{Error, Result as WindmillResult}; #[cfg(feature = "parquet")] use windmill_common::s3_helpers::build_object_store_client; use windmill_common::{ @@ -79,6 +81,7 @@ pub fn routes_global_service() -> Router { pub fn workspaced_service() -> Router { Router::new() .route("/create", post(create_trigger)) + .route("/create_many", post(create_many_http_trigger)) .route("/list", get(list_triggers)) .route("/get/*path", get(get_trigger)) .route("/update/*path", post(update_trigger)) @@ -273,7 +276,7 @@ async fn get_trigger( fn validate_authentication_method( authentication_method: AuthenticationMethod, raw_string: Option, -) -> error::Result<()> { +) -> WindmillResult<()> { match (authentication_method, raw_string) { (AuthenticationMethod::CustomScript, raw) if !raw.unwrap_or(false) == true => { return Err(Error::BadRequest( @@ -287,57 +290,21 @@ fn validate_authentication_method( Ok(()) } -async fn increase_trigger_version_and_commit(mut tx: PgTransaction<'_>) -> error::Result<()> { +async fn increase_trigger_version(tx: &mut PgConnection) -> WindmillResult<()> { sqlx::query!("SELECT nextval('http_trigger_version_seq')",) - .fetch_one(&mut *tx) + .fetch_one(tx) .await?; - tx.commit().await?; - Ok(()) } -async fn create_trigger( - authed: ApiAuthed, - Extension(db): Extension, - Extension(user_db): Extension, - Path(w_id): Path, - Json(ct): Json, -) -> error::Result<(StatusCode, String)> { - require_admin(authed.is_admin, &authed.username)?; - - if !VALID_ROUTE_PATH_RE.is_match(&ct.route_path) { - return Err(error::Error::BadRequest("Invalid route path".to_string())); - } - - validate_authentication_method(ct.authentication_method, ct.raw_string)?; - - // route path key is extracted from the route path to check for uniqueness - // it replaces /?:{key} with :key - // it will also remove the leading / if present, not an issue as we only allow : after slashes - let route_path_key = ROUTE_PATH_KEY_RE.replace_all(&ct.route_path, ":key"); - let exists = route_path_key_exists( - &route_path_key, - &ct.http_method, - &w_id, - None, - ct.workspaced_route, - &db, - ) - .await?; - if exists { - return Err(error::Error::BadRequest( - "A route already exists with this path".to_string(), - )); - } - - if *CLOUD_HOSTED && (ct.is_static_website || ct.static_asset_config.is_some()) { - return Err(error::Error::BadRequest( - "Static website and static asset are not supported on cloud".to_string(), - )); - } - - let mut tx = user_db.begin(&authed).await?; +async fn create_trigger_inner( + tx: &mut PgConnection, + w_id: &str, + authed: &ApiAuthed, + new_http_trigger: &NewTrigger, + route_path_key: &str, +) -> WindmillResult<()> { sqlx::query!( r#" INSERT INTO http_trigger ( @@ -365,51 +332,199 @@ async fn create_trigger( ) "#, w_id, - ct.path, - ct.route_path, + new_http_trigger.path, + new_http_trigger.route_path, &route_path_key, - ct.workspaced_route, - ct.authentication_resource_path, - ct.wrap_body.unwrap_or(false), - ct.raw_string.unwrap_or(false), - ct.script_path, - ct.is_flow, - ct.is_async, - ct.authentication_method as _, - ct.http_method as _, - ct.static_asset_config as _, + new_http_trigger.workspaced_route, + new_http_trigger.authentication_resource_path, + new_http_trigger.wrap_body.unwrap_or(false), + new_http_trigger.raw_string.unwrap_or(false), + new_http_trigger.script_path, + new_http_trigger.is_flow, + new_http_trigger.is_async, + new_http_trigger.authentication_method as _, + new_http_trigger.http_method as _, + new_http_trigger.static_asset_config as _, &authed.username, &authed.email, - ct.is_static_website + new_http_trigger.is_static_website ) .execute(&mut *tx) .await?; audit_log( &mut *tx, - &authed, + authed, "http_triggers.create", ActionKind::Create, &w_id, - Some(ct.path.as_str()), + Some(new_http_trigger.path.as_str()), None, ) .await?; - increase_trigger_version_and_commit(tx).await?; + increase_trigger_version(tx).await?; + + Ok(()) +} + +fn check_no_duplicates<'trigger>( + new_http_triggers: &[NewTrigger], + route_path_key: &[Cow<'trigger, str>], +) -> Result<(), Error> { + let mut seen = HashSet::with_capacity(new_http_triggers.len()); + + for (i, trigger) in new_http_triggers.iter().enumerate() { + if !seen.insert((&route_path_key[i], trigger.http_method, trigger.workspaced_route)) { + return Err(Error::BadRequest(format!( + "Duplicate HTTP route detected: '{}'. Each HTTP route must have a unique 'route_path'.", + &trigger.route_path + ))); + } + } + + Ok(()) +} + +async fn create_many_http_trigger( + authed: ApiAuthed, + Extension(db): Extension, + Extension(user_db): Extension, + Path(w_id): Path, + Json(new_http_triggers): Json>, +) -> WindmillResult<(StatusCode, String)> { + require_admin(authed.is_admin, &authed.username)?; + + let error_wrapper = |path: &str, error: Error| -> Error { + anyhow!( + "Error occurred for HTTP route at route path: {}, error: {}", + path, + error + ) + .into() + }; + + let mut route_path_keys = Vec::with_capacity(new_http_triggers.len()); + + for new_http_trigger in new_http_triggers.iter() { + let route_path_key = validate_http_trigger(&db, &w_id, new_http_trigger) + .await + .map_err(|err| error_wrapper(&new_http_trigger.route_path, err))?; + + route_path_keys.push(route_path_key); + } + + check_no_duplicates(&new_http_triggers, &route_path_keys)?; + + let mut tx = user_db.begin(&authed).await?; + + for (i, new_http_trigger) in new_http_triggers.iter().enumerate() { + create_trigger_inner( + &mut tx, + &w_id, + &authed, + new_http_trigger, + &route_path_keys[i], + ) + .await + .map_err(|err| error_wrapper(&new_http_trigger.route_path, err))?; + } + + tx.commit().await?; + + for http_trigger in new_http_triggers.into_iter() { + handle_deployment_metadata( + &authed.email, + &authed.username, + &db, + &w_id, + windmill_git_sync::DeployedObject::HttpTrigger { path: http_trigger.path.clone() }, + Some(format!("HTTP route '{}' created", http_trigger.path)), + true, + ) + .await?; + } + + Ok((StatusCode::CREATED, format!("Created all HTTP routes"))) +} + +async fn validate_http_trigger<'trigger>( + db: &DB, + w_id: &str, + new_http_trigger: &'trigger NewTrigger, +) -> WindmillResult> { + if !VALID_ROUTE_PATH_RE.is_match(&new_http_trigger.route_path) { + return Err(error::Error::BadRequest("Invalid route path".to_string())); + } + + validate_authentication_method( + new_http_trigger.authentication_method, + new_http_trigger.raw_string, + )?; + + // route path key is extracted from the route path to check for uniqueness + // it replaces /?:{key} with :key + // it will also remove the leading / if present, not an issue as we only allow : after slashes + let route_path_key = ROUTE_PATH_KEY_RE.replace_all(&new_http_trigger.route_path, ":key"); + + let exists = route_path_key_exists( + &route_path_key, + &new_http_trigger.http_method, + &w_id, + None, + new_http_trigger.workspaced_route, + db, + ) + .await?; + + if exists { + return Err(error::Error::BadRequest( + "A route already exists with this path".to_string(), + )); + } + + if *CLOUD_HOSTED + && (new_http_trigger.is_static_website || new_http_trigger.static_asset_config.is_some()) + { + return Err(error::Error::BadRequest( + "Static website and static asset are not supported on cloud".to_string(), + )); + } + + Ok(route_path_key) +} + +async fn create_trigger( + authed: ApiAuthed, + Extension(db): Extension, + Extension(user_db): Extension, + Path(w_id): Path, + Json(new_http_trigger): Json, +) -> WindmillResult<(StatusCode, String)> { + require_admin(authed.is_admin, &authed.username)?; + + let route_path_key = validate_http_trigger(&db, &w_id, &new_http_trigger).await?; + + let mut tx = user_db.begin(&authed).await?; + + let http_trigger_path = new_http_trigger.path.clone(); + + create_trigger_inner(&mut tx, &w_id, &authed, &new_http_trigger, &route_path_key).await?; + + tx.commit().await?; handle_deployment_metadata( &authed.email, &authed.username, &db, &w_id, - windmill_git_sync::DeployedObject::HttpTrigger { path: ct.path.clone() }, - Some(format!("HTTP trigger '{}' created", ct.path)), + windmill_git_sync::DeployedObject::HttpTrigger { path: new_http_trigger.path.clone() }, + Some(format!("HTTP route '{}' created", new_http_trigger.path)), true, ) .await?; - Ok((StatusCode::CREATED, format!("{}", ct.path))) + Ok((StatusCode::CREATED, format!("{}", http_trigger_path))) } async fn update_trigger( @@ -418,8 +533,9 @@ async fn update_trigger( Extension(db): Extension, Path((w_id, path)): Path<(String, StripPath)>, Json(ct): Json, -) -> error::Result { +) -> WindmillResult { let path = path.to_path(); + if *CLOUD_HOSTED && (ct.is_static_website || ct.static_asset_config.is_some()) { return Err(error::Error::BadRequest( "Static website and static asset are not supported on cloud".to_string(), @@ -563,7 +679,9 @@ async fn update_trigger( ) .await?; - increase_trigger_version_and_commit(tx).await?; + increase_trigger_version(&mut tx).await?; + + tx.commit().await?; handle_deployment_metadata( &authed.email, @@ -571,7 +689,7 @@ async fn update_trigger( &db, &w_id, windmill_git_sync::DeployedObject::HttpTrigger { path: ct.path.clone() }, - Some(format!("HTTP trigger '{}' updated", ct.path)), + Some(format!("HTTP route '{}' updated", ct.path)), true, ) .await?; @@ -584,7 +702,7 @@ async fn delete_trigger( Extension(db): Extension, Extension(user_db): Extension, Path((w_id, path)): Path<(String, StripPath)>, -) -> error::Result { +) -> WindmillResult { require_admin(authed.is_admin, &authed.username)?; let path = path.to_path(); let mut tx = user_db.begin(&authed).await?; @@ -609,7 +727,9 @@ async fn delete_trigger( ) .await?; - increase_trigger_version_and_commit(tx).await?; + increase_trigger_version(&mut tx).await?; + + tx.commit().await?; handle_deployment_metadata( &authed.email, @@ -617,12 +737,12 @@ async fn delete_trigger( &db, &w_id, windmill_git_sync::DeployedObject::HttpTrigger { path: path.to_string() }, - Some(format!("HTTP trigger '{}' deleted", path)), + Some(format!("HTTP route '{}' deleted", path)), true, ) .await?; - Ok(format!("HTTP trigger {path} deleted")) + Ok(format!("HTTP route {path} deleted")) } async fn exists_trigger( @@ -660,7 +780,7 @@ async fn route_path_key_exists( trigger_path: Option<&str>, workspaced_route: Option, db: &DB, -) -> error::Result { +) -> WindmillResult { let exists = if *CLOUD_HOSTED { sqlx::query_scalar!( r#" @@ -861,7 +981,7 @@ pub async fn refresh_routers(db: &DB) -> Result<(bool, RwLockReadGuard<'_, Route .insert(format!("{}/*wm_subpath", full_path), trigger.clone()) .unwrap_or_else(|e| { tracing::warn!( - "Failed to consider http trigger route {}/*wm_subpath: {:?}", + "Failed to consider HTTP route {}/*wm_subpath: {:?}", full_path, e, ); @@ -870,11 +990,7 @@ pub async fn refresh_routers(db: &DB) -> Result<(bool, RwLockReadGuard<'_, Route router .insert(full_path.clone(), trigger.clone()) .unwrap_or_else(|e| { - tracing::warn!( - "Failed to consider http trigger route {}: {:?}", - full_path, - e, - ); + tracing::warn!("Failed to consider HTTP route {}: {:?}", full_path, e,); }); } @@ -898,7 +1014,7 @@ async fn get_http_route_trigger( db: &DB, user_db: UserDB, method: &http::Method, -) -> error::Result<(TriggerRoute, String, HashMap, ApiAuthed)> { +) -> WindmillResult<(TriggerRoute, String, HashMap, ApiAuthed)> { let http_method: HttpMethod = method.try_into()?; let requested_path = format!("/{}", route_path); @@ -947,11 +1063,11 @@ async fn get_http_route_trigger( ); let exists = match HTTP_ACCESS_CACHE.get(&cache_key) { Some(cache_entry) if cache_entry.expiry > std::time::Instant::now() => { - tracing::debug!("HTTP access cache hit for trigger {}", trigger.path); + tracing::debug!("HTTP access cache hit for route {}", trigger.path); true } _ => { - tracing::debug!("HTTP access cache miss for trigger {}", trigger.path); + tracing::debug!("HTTP access cache miss for route {}", trigger.path); let mut tx = user_db.begin(&authed).await?; let exists = sqlx::query_scalar!( r#" @@ -1002,7 +1118,7 @@ async fn get_http_route_trigger( trigger.email.clone(), &trigger.workspace_id, &db, - Some(username_override.unwrap_or(format!("http-{}", trigger.path))), + Some(username_override.unwrap_or(format!("HTTP-{}", trigger.path))), ) .await?; @@ -1030,6 +1146,14 @@ async fn route_job( .await .map_err(|e| e.into_response())?; + if trigger.script_path.is_empty() && trigger.static_asset_config.is_none() { + return Err(Error::BadRequest(format!( + "Script path of HTTP route at path: {} must not be empty", + trigger.path + )) + .into_response()); + } + let args = args .process_args( &authed, @@ -1066,11 +1190,11 @@ async fn route_job( let authentication_method = match HTTP_AUTH_CACHE.get(&cache_key) { Some(cache_entry) if cache_entry.expiry > std::time::Instant::now() => { - tracing::debug!("HTTP auth method cache hit for trigger {}", trigger.path); + tracing::debug!("HTTP auth method cache hit for route {}", trigger.path); cache_entry.value } _ => { - tracing::debug!("HTTP auth method cache miss for trigger {}", trigger.path); + tracing::debug!("HTTP auth method cache miss for route {}", trigger.path); let auth_method = try_get_resource_from_db_as::< crate::http_trigger_auth::AuthenticationMethod, >( diff --git a/backend/windmill-api/src/indexer_ee.rs b/backend/windmill-api/src/indexer_ee.rs deleted file mode 100644 index 2ccca92c27..0000000000 --- a/backend/windmill-api/src/indexer_ee.rs +++ /dev/null @@ -1,9 +0,0 @@ -use axum::Router; - -pub fn workspaced_service() -> Router { - Router::new() -} - -pub fn global_service() -> Router { - Router::new() -} diff --git a/backend/windmill-api/src/indexer_oss.rs b/backend/windmill-api/src/indexer_oss.rs new file mode 100644 index 0000000000..eee87acdcb --- /dev/null +++ b/backend/windmill-api/src/indexer_oss.rs @@ -0,0 +1,16 @@ +#[cfg(feature = "private")] +#[allow(unused)] +pub use crate::indexer_ee::*; + +#[cfg(not(feature = "private"))] +use axum::Router; + +#[cfg(not(feature = "private"))] +pub fn workspaced_service() -> Router { + Router::new() +} + +#[cfg(not(feature = "private"))] +pub fn global_service() -> Router { + Router::new() +} diff --git a/backend/windmill-api/src/inkeep_oss.rs b/backend/windmill-api/src/inkeep_oss.rs new file mode 100644 index 0000000000..ca1ffebd21 --- /dev/null +++ b/backend/windmill-api/src/inkeep_oss.rs @@ -0,0 +1,11 @@ +#[cfg(feature = "private")] +#[allow(unused)] +pub use crate::inkeep_ee::*; + +#[cfg(not(feature = "private"))] +use axum::Router; + +#[cfg(not(feature = "private"))] +pub fn global_service() -> Router { + Router::new() +} diff --git a/backend/windmill-api/src/job_helpers_ee.rs b/backend/windmill-api/src/job_helpers_oss.rs similarity index 70% rename from backend/windmill-api/src/job_helpers_ee.rs rename to backend/windmill-api/src/job_helpers_oss.rs index 4e14d16c5f..53d4a796a1 100644 --- a/backend/windmill-api/src/job_helpers_ee.rs +++ b/backend/windmill-api/src/job_helpers_oss.rs @@ -1,34 +1,45 @@ +#[cfg(feature = "private")] +#[allow(unused)] +pub use crate::job_helpers_ee::*; + +#[cfg(not(feature = "private"))] use axum::Router; +#[cfg(not(feature = "private"))] use serde::Serialize; +#[cfg(not(feature = "private"))] use uuid::Uuid; +#[cfg(not(feature = "private"))] use windmill_common::s3_helpers::StorageResourceType; -#[cfg(feature = "parquet")] +#[cfg(all(feature = "parquet", not(feature = "private")))] use crate::db::{ApiAuthed, DB}; -#[cfg(feature = "parquet")] +#[cfg(all(feature = "parquet", not(feature = "private")))] use object_store::{ObjectStore, PutMultipartOpts}; -#[cfg(feature = "parquet")] +#[cfg(all(feature = "parquet", not(feature = "private")))] use std::sync::Arc; +#[cfg(not(feature = "private"))] use windmill_common::error; -#[cfg(feature = "parquet")] +#[cfg(all(feature = "parquet", not(feature = "private")))] use windmill_common::{db::UserDB, s3_helpers::ObjectStoreResource}; -#[cfg(feature = "parquet")] +#[cfg(all(feature = "parquet", not(feature = "private")))] use bytes::Bytes; -#[cfg(feature = "parquet")] +#[cfg(all(feature = "parquet", not(feature = "private")))] use futures::Stream; -#[cfg(feature = "parquet")] +#[cfg(all(feature = "parquet", not(feature = "private")))] use axum::response::Response; -#[cfg(feature = "parquet")] +#[cfg(all(feature = "parquet", not(feature = "private")))] use serde::Deserialize; #[derive(Serialize)] +#[cfg(not(feature = "private"))] pub struct UploadFileResponse { pub file_key: String, } #[derive(Deserialize)] +#[cfg(not(feature = "private"))] pub struct LoadImagePreviewQuery { #[allow(dead_code)] pub file_key: String, @@ -37,6 +48,7 @@ pub struct LoadImagePreviewQuery { } #[derive(Deserialize)] +#[cfg(not(feature = "private"))] pub struct DownloadFileQuery { #[allow(dead_code)] pub file_key: String, @@ -46,11 +58,12 @@ pub struct DownloadFileQuery { pub s3_resource_path: Option, } +#[cfg(not(feature = "private"))] pub fn workspaced_service() -> Router { Router::new() } -#[cfg(feature = "parquet")] +#[cfg(all(feature = "parquet", not(feature = "private")))] pub async fn get_workspace_s3_resource<'c>( _authed: &ApiAuthed, _db: &DB, @@ -63,10 +76,12 @@ pub async fn get_workspace_s3_resource<'c>( Ok((None, None)) } +#[cfg(not(feature = "private"))] pub fn get_random_file_name(_file_extension: Option) -> String { unimplemented!("Not implemented in Windmill's Open Source repository") } +#[cfg(not(feature = "private"))] pub async fn get_s3_resource<'c>( _authed: &ApiAuthed, _db: &DB, @@ -82,7 +97,7 @@ pub async fn get_s3_resource<'c>( )) } -#[cfg(feature = "parquet")] +#[cfg(all(feature = "parquet", not(feature = "private")))] pub async fn upload_file_from_req( _s3_client: Arc, _file_key: &str, @@ -94,7 +109,7 @@ pub async fn upload_file_from_req( )) } -#[cfg(feature = "parquet")] +#[cfg(all(feature = "parquet", not(feature = "private")))] pub async fn upload_file_internal( _s3_client: Arc, _file_key: &str, @@ -106,7 +121,7 @@ pub async fn upload_file_internal( )) } -#[cfg(feature = "parquet")] +#[cfg(all(feature = "parquet", not(feature = "private")))] pub async fn download_s3_file_internal( _authed: ApiAuthed, _db: &DB, diff --git a/backend/windmill-api/src/jobs.rs b/backend/windmill-api/src/jobs.rs index cf9e7d06de..8b0b5e5e76 100644 --- a/backend/windmill-api/src/jobs.rs +++ b/backend/windmill-api/src/jobs.rs @@ -26,6 +26,7 @@ use tokio::io::AsyncReadExt; #[cfg(feature = "prometheus")] use tokio::time::Instant; use tower::ServiceBuilder; +use windmill_common::auth::is_super_admin_email; use windmill_common::error::JsonResult; use windmill_common::flow_status::{JobResult, RestartedFrom}; use windmill_common::jobs::{format_completed_job_result, format_result, ENTRYPOINT_OVERRIDE}; @@ -64,7 +65,7 @@ use sqlx::types::JsonRawValue; use sqlx::{types::Uuid, FromRow, Postgres, Transaction}; use tower_http::cors::{Any, CorsLayer}; use urlencoding::encode; -use windmill_audit::audit_ee::{audit_log, AuditAuthor}; +use windmill_audit::audit_oss::{audit_log, AuditAuthor}; use windmill_audit::ActionKind; use windmill_common::worker::{to_raw_value, CUSTOM_TAGS_PER_WORKSPACE}; use windmill_common::{ @@ -3140,28 +3141,27 @@ pub fn add_raw_string( } async fn check_tag_available_for_workspace( + db: &DB, w_id: &str, tag: &Option, authed: &ApiAuthed, ) -> error::Result<()> { if let Some(tag) = tag { - if tag == "" { + if tag.is_empty() { return Ok(()); } let tags = get_scope_tags(authed); + let mut is_tag_available_in_workspace = None; + let mut is_tag_in_workspace_custom_tags = false; - if let Some(tags) = tags { - if !tags.contains(&tag.as_str()) { - return Err(Error::BadRequest(format!( - "Tag {tag} is not available in your scope" - ))); - } + if let Some(tags) = tags.as_ref() { + is_tag_available_in_workspace = Some(tags.contains(&tag.as_str())); } let custom_tags_per_w = CUSTOM_TAGS_PER_WORKSPACE.read().await; if custom_tags_per_w.0.contains(&tag.to_string()) { - Ok(()) + is_tag_in_workspace_custom_tags = true; } else if custom_tags_per_w.1.contains_key(tag) && custom_tags_per_w .1 @@ -3169,21 +3169,38 @@ async fn check_tag_available_for_workspace( .unwrap() .contains(&w_id.to_string()) { - Ok(()) - } else { + is_tag_in_workspace_custom_tags = true; + } + + match is_tag_available_in_workspace { + Some(true) | None => { + if is_tag_in_workspace_custom_tags { + return Ok(()); + } + } + _ => {} + } + + if !is_super_admin_email(db, &authed.email).await? { + if tags.is_some() && is_tag_available_in_workspace.is_some() { + return Err(Error::BadRequest(format!( + "Tag {tag} is not available in your scope" + ))); + } + return Err(error::Error::BadRequest(format!( - "Tag {tag} cannot be used on workspace {w_id}: (CUSTOM_TAGS: {:?})", + "Only super admins are allowed to use tags that are not included in the allowed CUSTOM_TAGS: {:?}", custom_tags_per_w ))); } - } else { - Ok(()) } + + return Ok(()); } #[cfg(feature = "enterprise")] pub async fn check_license_key_valid() -> error::Result<()> { - use windmill_common::ee::LICENSE_KEY_VALID; + use windmill_common::ee_oss::LICENSE_KEY_VALID; let valid = *LICENSE_KEY_VALID.read().await; if !valid { @@ -3509,7 +3526,7 @@ pub async fn run_flow_by_path_inner( let tag = run_query.tag.clone().or(tag); - check_tag_available_for_workspace(&w_id, &tag, &authed).await?; + check_tag_available_for_workspace(&db, &w_id, &tag, &authed).await?; let scheduled_for = run_query.get_scheduled_for(&db).await?; let (email, permissioned_as, push_authed, tx) = @@ -3701,7 +3718,7 @@ pub async fn run_script_by_path_inner( let scheduled_for = run_query.get_scheduled_for(&db).await?; let tag = run_query.tag.clone().or(tag); - check_tag_available_for_workspace(&w_id, &tag, &authed).await?; + check_tag_available_for_workspace(&db, &w_id, &tag, &authed).await?; let (email, permissioned_as, push_authed, tx) = if let Some(on_behalf_of) = on_behalf_of.as_ref() { @@ -3772,7 +3789,7 @@ pub async fn run_workflow_as_code( #[cfg(feature = "enterprise")] check_license_key_valid().await?; - check_tag_available_for_workspace(&w_id, &run_query.tag, &authed).await?; + check_tag_available_for_workspace(&db, &w_id, &run_query.tag, &authed).await?; if *CLOUD_HOSTED { tracing::info!("workflow_as_code_tracing id {i} "); @@ -4369,7 +4386,7 @@ pub async fn run_wait_result_job_by_path_get( drop(tx); let tag = run_query.tag.clone().or(tag); - check_tag_available_for_workspace(&w_id, &tag, &authed).await?; + check_tag_available_for_workspace(&db, &w_id, &tag, &authed).await?; let (email, permissioned_as, push_authed, tx) = if let Some(on_behalf_of) = on_behalf_authed.as_ref() { @@ -4509,7 +4526,7 @@ pub async fn run_wait_result_script_by_path_internal( drop(tx); let tag = run_query.tag.clone().or(tag); - check_tag_available_for_workspace(&w_id, &tag, &authed).await?; + check_tag_available_for_workspace(&db, &w_id, &tag, &authed).await?; let (email, permissioned_as, push_authed, tx) = if let Some(on_behalf_of) = on_behalf_of.as_ref() { @@ -4610,7 +4627,7 @@ pub async fn run_wait_result_script_by_hash( check_scopes(&authed, || format!("run:script/{path}"))?; let tag = run_query.tag.clone().or(tag); - check_tag_available_for_workspace(&w_id, &tag, &authed).await?; + check_tag_available_for_workspace(&db, &w_id, &tag, &authed).await?; let (email, permissioned_as, push_authed, tx) = if let Some(email) = on_behalf_of_email.as_ref() { @@ -4730,7 +4747,7 @@ pub async fn run_wait_result_flow_by_path_internal( drop(tx); let tag = run_query.tag.clone().or(tag); - check_tag_available_for_workspace(&w_id, &tag, &authed).await?; + check_tag_available_for_workspace(&db, &w_id, &tag, &authed).await?; let (email, permissioned_as, push_authed, tx) = if let Some(on_behalf_of_email) = on_behalf_of_email.as_ref() { @@ -4804,7 +4821,7 @@ async fn run_preview_script( } let scheduled_for = run_query.get_scheduled_for(&db).await?; let tag = run_query.tag.clone().or(preview.tag.clone()); - check_tag_available_for_workspace(&w_id, &tag, &authed).await?; + check_tag_available_for_workspace(&db, &w_id, &tag, &authed).await?; let tx = PushIsolationLevel::Isolated(user_db.clone(), authed.clone().into()); let (uuid, tx) = push( @@ -4886,7 +4903,7 @@ async fn run_bundle_preview_script( let scheduled_for = run_query.get_scheduled_for(&db).await?; let tag = run_query.tag.clone().or(preview.tag.clone()); - check_tag_available_for_workspace(&w_id, &tag, &authed).await?; + check_tag_available_for_workspace(&db, &w_id, &tag, &authed).await?; let ltx = PushIsolationLevel::Isolated(user_db.clone(), authed.clone().into()); let args = preview.args.unwrap_or_default(); @@ -5470,7 +5487,7 @@ async fn run_preview_flow_job( } let scheduled_for = run_query.get_scheduled_for(&db).await?; let tag = run_query.tag.clone().or(raw_flow.tag.clone()); - check_tag_available_for_workspace(&w_id, &tag, &authed).await?; + check_tag_available_for_workspace(&db, &w_id, &tag, &authed).await?; let tx = PushIsolationLevel::Isolated(user_db.clone(), authed.clone().into()); let (uuid, tx) = push( @@ -5565,7 +5582,7 @@ pub async fn run_job_by_hash_inner( let scheduled_for = run_query.get_scheduled_for(&db).await?; let tag = run_query.tag.clone().or(tag); - check_tag_available_for_workspace(&w_id, &tag, &authed).await?; + check_tag_available_for_workspace(&db, &w_id, &tag, &authed).await?; let (email, permissioned_as, push_authed, tx) = if let Some(email) = on_behalf_of_email.as_ref() { diff --git a/backend/windmill-api/src/kafka_triggers_ee.rs b/backend/windmill-api/src/kafka_triggers_oss.rs similarity index 76% rename from backend/windmill-api/src/kafka_triggers_ee.rs rename to backend/windmill-api/src/kafka_triggers_oss.rs index 0a24151ae3..9d698e2fbf 100644 --- a/backend/windmill-api/src/kafka_triggers_ee.rs +++ b/backend/windmill-api/src/kafka_triggers_oss.rs @@ -1,14 +1,24 @@ +#[cfg(feature = "private")] +#[allow(unused)] +pub use crate::kafka_triggers_ee::*; + +#[cfg(not(feature = "private"))] use crate::db::DB; +#[cfg(not(feature = "private"))] use axum::Router; +#[cfg(not(feature = "private"))] use serde::{Deserialize, Serialize}; #[derive(Serialize, Deserialize)] +#[cfg(not(feature = "private"))] pub struct KafkaResourceSecurity {} +#[cfg(not(feature = "private"))] pub fn workspaced_service() -> Router { Router::new() } +#[cfg(not(feature = "private"))] pub fn start_kafka_consumers( _db: DB, mut _killpill_rx: tokio::sync::broadcast::Receiver<()>, @@ -17,9 +27,11 @@ pub fn start_kafka_consumers( } #[derive(Serialize, Deserialize)] +#[cfg(not(feature = "private"))] pub enum KafkaTriggerConfigConnection {} #[derive(Serialize, Clone)] +#[cfg(not(feature = "private"))] pub struct KafkaTrigger { pub workspace_id: String, pub path: String, @@ -39,4 +51,4 @@ pub struct KafkaTrigger { #[serde(skip_serializing_if = "Option::is_none")] pub error: Option, pub enabled: bool, -} \ No newline at end of file +} diff --git a/backend/windmill-api/src/lib.rs b/backend/windmill-api/src/lib.rs index d79639f0ab..8a42902508 100644 --- a/backend/windmill-api/src/lib.rs +++ b/backend/windmill-api/src/lib.rs @@ -8,15 +8,15 @@ use crate::db::ApiAuthed; #[cfg(feature = "enterprise")] -use crate::ee::ExternalJwks; +use crate::ee_oss::ExternalJwks; #[cfg(feature = "embedding")] use crate::embeddings::load_embeddings_db; #[cfg(feature = "oauth2")] -use crate::oauth2_ee::AllClients; +use crate::oauth2_oss::AllClients; #[cfg(feature = "oauth2")] -use crate::oauth2_ee::SlackVerifier; +use crate::oauth2_oss::SlackVerifier; #[cfg(feature = "smtp")] -use crate::smtp_server_ee::SmtpServer; +use crate::smtp_server_oss::SmtpServer; #[cfg(feature = "mcp")] use crate::mcp::{setup_mcp_server, Runner as McpRunner}; @@ -28,7 +28,7 @@ use crate::{ }; #[cfg(feature = "agent_worker_server")] -use agent_workers_ee::AgentCache; +use agent_workers_oss::AgentCache; use anyhow::Context; use argon2::Argon2; @@ -58,22 +58,26 @@ use windmill_common::db::UserDB; use windmill_common::worker::CLOUD_HOSTED; use windmill_common::{utils::GIT_VERSION, BASE_URL, INSTANCE_NAME}; -use crate::scim_ee::has_scim_token; +use crate::scim_oss::has_scim_token; use windmill_common::error::AppError; +#[cfg(all(feature = "agent_worker_server", feature = "private"))] +pub mod agent_workers_ee; #[cfg(feature = "agent_worker_server")] -mod agent_workers_ee; +mod agent_workers_oss; mod ai; mod apps; pub mod args; mod audit; -mod auth; +pub mod auth; mod capture; mod concurrency_groups; mod configs; mod db; mod drafts; +#[cfg(feature = "private")] pub mod ee; +pub mod ee_oss; pub mod embeddings; mod favorite; mod flows; @@ -86,56 +90,93 @@ mod http_trigger_args; mod http_trigger_auth; #[cfg(feature = "http_trigger")] pub mod http_triggers; -mod indexer_ee; +#[cfg(feature = "private")] +pub mod indexer_ee; +mod indexer_oss; +#[cfg(feature = "private")] +mod inkeep_ee; +mod inkeep_oss; mod inputs; mod integration; #[cfg(feature = "postgres_trigger")] mod postgres_triggers; mod approvals; +#[cfg(all(feature = "enterprise", feature = "private"))] +pub mod apps_ee; #[cfg(feature = "enterprise")] -mod apps_ee; +mod apps_oss; +#[cfg(all(feature = "enterprise", feature = "gcp_trigger", feature = "private"))] +pub mod gcp_triggers_ee; #[cfg(all(feature = "enterprise", feature = "gcp_trigger"))] -mod gcp_triggers_ee; +mod gcp_triggers_oss; +#[cfg(all(feature = "enterprise", feature = "private"))] +pub mod git_sync_ee; #[cfg(feature = "enterprise")] -mod git_sync_ee; +mod git_sync_oss; +#[cfg(all(feature = "parquet", feature = "private"))] +pub mod job_helpers_ee; #[cfg(feature = "parquet")] -mod job_helpers_ee; +mod job_helpers_oss; pub mod job_metrics; pub mod jobs; +#[cfg(all(feature = "enterprise", feature = "kafka", feature = "private"))] +pub mod kafka_triggers_ee; #[cfg(all(feature = "enterprise", feature = "kafka"))] -mod kafka_triggers_ee; +mod kafka_triggers_oss; #[cfg(feature = "mqtt_trigger")] mod mqtt_triggers; +#[cfg(all(feature = "enterprise", feature = "nats", feature = "private"))] +pub mod nats_triggers_ee; #[cfg(all(feature = "enterprise", feature = "nats"))] -mod nats_triggers_ee; -#[cfg(feature = "oauth2")] +mod nats_triggers_oss; +#[cfg(all(feature = "oauth2", feature = "private"))] pub mod oauth2_ee; -mod oidc_ee; +#[cfg(feature = "oauth2")] +pub mod oauth2_oss; +#[cfg(feature = "private")] +pub mod oidc_ee; +mod oidc_oss; mod raw_apps; mod resources; -mod saml_ee; +#[cfg(feature = "private")] +pub mod saml_ee; +mod saml_oss; mod schedule; -mod scim_ee; +#[cfg(feature = "private")] +pub mod scim_ee; +mod scim_oss; mod scripts; mod service_logs; mod settings; mod slack_approvals; +#[cfg(all(feature = "smtp", feature = "private"))] +pub mod smtp_server_ee; #[cfg(feature = "smtp")] -mod smtp_server_ee; +mod smtp_server_oss; +#[cfg(all(feature = "enterprise", feature = "sqs_trigger", feature = "private"))] +pub mod sqs_triggers_ee; #[cfg(all(feature = "enterprise", feature = "sqs_trigger"))] -mod sqs_triggers_ee; -mod teams_approvals_ee; +mod sqs_triggers_oss; +#[cfg(feature = "private")] +pub mod teams_approvals_ee; +mod teams_approvals_oss; mod trigger_helpers; mod static_assets; +#[cfg(all(feature = "stripe", feature = "enterprise", feature = "private"))] +pub mod stripe_ee; #[cfg(all(feature = "stripe", feature = "enterprise"))] -mod stripe_ee; -mod teams_ee; +mod stripe_oss; +#[cfg(feature = "private")] +pub mod teams_ee; +mod teams_oss; mod tracing_init; mod triggers; mod users; -mod users_ee; +#[cfg(feature = "private")] +pub mod users_ee; +mod users_oss; mod utils; mod variables; pub mod webhook_util; @@ -143,9 +184,11 @@ pub mod webhook_util; mod websocket_triggers; mod workers; mod workspaces; -mod workspaces_ee; +#[cfg(feature = "private")] +pub mod workspaces_ee; mod workspaces_export; mod workspaces_extra; +mod workspaces_oss; #[cfg(feature = "mcp")] mod mcp; @@ -218,9 +261,9 @@ type IndexReader = (); type ServiceLogIndexReader = (); #[cfg(feature = "tantivy")] -type IndexReader = windmill_indexer::completed_runs_ee::IndexReader; +type IndexReader = windmill_indexer::completed_runs_oss::IndexReader; #[cfg(feature = "tantivy")] -type ServiceLogIndexReader = windmill_indexer::service_logs_ee::ServiceLogIndexReader; +type ServiceLogIndexReader = windmill_indexer::service_logs_oss::ServiceLogIndexReader; pub async fn run_server( db: DB, @@ -278,7 +321,7 @@ pub async fn run_server( .allow_headers([http::header::CONTENT_TYPE, http::header::AUTHORIZATION]) .allow_origin(Any); - let sp_extension = Arc::new(saml_ee::build_sp_extension().await?); + let sp_extension = Arc::new(saml_oss::build_sp_extension().await?); if server_mode { #[cfg(feature = "embedding")] @@ -317,7 +360,7 @@ pub async fn run_server( let job_helpers_service = { #[cfg(feature = "parquet")] { - job_helpers_ee::workspaced_service() + job_helpers_oss::workspaced_service() } #[cfg(not(feature = "parquet"))] @@ -329,7 +372,7 @@ pub async fn run_server( let kafka_triggers_service = { #[cfg(all(feature = "enterprise", feature = "kafka"))] { - kafka_triggers_ee::workspaced_service() + kafka_triggers_oss::workspaced_service() } #[cfg(not(all(feature = "enterprise", feature = "kafka")))] @@ -341,7 +384,7 @@ pub async fn run_server( let nats_triggers_service = { #[cfg(all(feature = "enterprise", feature = "nats"))] { - nats_triggers_ee::workspaced_service() + nats_triggers_oss::workspaced_service() } #[cfg(not(all(feature = "enterprise", feature = "nats")))] @@ -365,7 +408,7 @@ pub async fn run_server( let gcp_triggers_service = { #[cfg(all(feature = "enterprise", feature = "gcp_trigger"))] { - gcp_triggers_ee::workspaced_service() + gcp_triggers_oss::workspaced_service() } #[cfg(not(all(feature = "enterprise", feature = "gcp_trigger")))] @@ -377,7 +420,7 @@ pub async fn run_server( let sqs_triggers_service = { #[cfg(all(feature = "enterprise", feature = "sqs_trigger"))] { - sqs_triggers_ee::workspaced_service() + sqs_triggers_oss::workspaced_service() } #[cfg(not(all(feature = "enterprise", feature = "sqs_trigger")))] @@ -432,13 +475,13 @@ pub async fn run_server( #[cfg(all(feature = "enterprise", feature = "kafka"))] { let kafka_killpill_rx = killpill_rx.resubscribe(); - kafka_triggers_ee::start_kafka_consumers(db.clone(), kafka_killpill_rx); + kafka_triggers_oss::start_kafka_consumers(db.clone(), kafka_killpill_rx); } #[cfg(all(feature = "enterprise", feature = "nats"))] { let nats_killpill_rx = killpill_rx.resubscribe(); - nats_triggers_ee::start_nats_consumers(db.clone(), nats_killpill_rx); + nats_triggers_oss::start_nats_consumers(db.clone(), nats_killpill_rx); } #[cfg(feature = "postgres_trigger")] @@ -456,13 +499,13 @@ pub async fn run_server( #[cfg(all(feature = "enterprise", feature = "sqs_trigger"))] { let sqs_killpill_rx = killpill_rx.resubscribe(); - sqs_triggers_ee::start_sqs(db.clone(), sqs_killpill_rx); + sqs_triggers_oss::start_sqs(db.clone(), sqs_killpill_rx); } #[cfg(all(feature = "enterprise", feature = "gcp_trigger"))] { let gcp_killpill_rx = killpill_rx.resubscribe(); - gcp_triggers_ee::start_consuming_gcp_pubsub_event(db.clone(), gcp_killpill_rx); + gcp_triggers_oss::start_consuming_gcp_pubsub_event(db.clone(), gcp_killpill_rx); } } @@ -497,7 +540,7 @@ pub async fn run_server( #[cfg(feature = "agent_worker_server")] let (agent_workers_router, agent_workers_bg_processor, agent_workers_killpill_tx) = if server_mode { - agent_workers_ee::workspaced_service(db.clone(), _base_internal_url.clone()) + agent_workers_oss::workspaced_service(db.clone(), _base_internal_url.clone()) } else { (Router::new(), vec![], None) }; @@ -535,7 +578,7 @@ pub async fn run_server( .nest("/oauth", { #[cfg(feature = "oauth2")] { - oauth2_ee::workspaced_service() + oauth2_oss::workspaced_service() } #[cfg(not(feature = "oauth2"))] @@ -552,7 +595,7 @@ pub async fn run_server( ) .nest("/variables", variables::workspaced_service()) .nest("/workspaces", workspaces::workspaced_service()) - .nest("/oidc", oidc_ee::workspaced_service()) + .nest("/oidc", oidc_oss::workspaced_service()) .nest("/http_triggers", http_triggers_service) .nest("/websocket_triggers", websocket_triggers_service) .nest("/kafka_triggers", kafka_triggers_service) @@ -579,22 +622,23 @@ pub async fn run_server( .nest("/schedules", schedule::global_service()) .nest("/embeddings", embeddings::global_service()) .nest("/ai", ai::global_service()) + .nest("/inkeep", inkeep_oss::global_service()) .route_layer(from_extractor::()) .route_layer(from_extractor::()) .nest("/jobs", jobs::global_root_service()) .nest( "/srch/w/:workspace_id/index", - indexer_ee::workspaced_service(), + indexer_oss::workspaced_service(), ) - .nest("/srch/index", indexer_ee::global_service()) - .nest("/oidc", oidc_ee::global_service()) + .nest("/srch/index", indexer_oss::global_service()) + .nest("/oidc", oidc_oss::global_service()) .nest( "/saml", - saml_ee::global_service().layer(Extension(Arc::clone(&sp_extension))), + saml_oss::global_service().layer(Extension(Arc::clone(&sp_extension))), ) .nest( "/scim", - scim_ee::global_service() + scim_oss::global_service() .route_layer(axum::middleware::from_fn(has_scim_token)), ) .nest("/concurrency_groups", concurrency_groups::global_service()) @@ -602,7 +646,7 @@ pub async fn run_server( .nest("/apps_u", { #[cfg(feature = "enterprise")] { - apps_ee::global_unauthed_service() + apps_oss::global_unauthed_service() } #[cfg(not(feature = "enterprise"))] @@ -621,7 +665,7 @@ pub async fn run_server( .nest("/agent_workers", { #[cfg(feature = "agent_worker_server")] { - agent_workers_ee::global_service().layer(Extension(agent_cache.clone())) + agent_workers_oss::global_service().layer(Extension(agent_cache.clone())) } #[cfg(not(feature = "agent_worker_server"))] { @@ -646,7 +690,7 @@ pub async fn run_server( .nest("/teams", { #[cfg(feature = "enterprise")] { - teams_ee::teams_service() + teams_oss::teams_service() } #[cfg(not(feature = "enterprise"))] @@ -660,12 +704,12 @@ pub async fn run_server( ) .route( "/w/:workspace_id/jobs/teams_approval/:job_id", - get(teams_approvals_ee::request_teams_approval), + get(teams_approvals_oss::request_teams_approval), ) .nest("/w/:workspace_id/github_app", { #[cfg(feature = "enterprise")] { - git_sync_ee::workspaced_service() + git_sync_oss::workspaced_service() } #[cfg(not(feature = "enterprise"))] @@ -674,7 +718,7 @@ pub async fn run_server( .nest("/github_app", { #[cfg(feature = "enterprise")] { - git_sync_ee::global_service() + git_sync_oss::global_service() } #[cfg(not(feature = "enterprise"))] @@ -695,7 +739,7 @@ pub async fn run_server( .nest("/oauth", { #[cfg(feature = "oauth2")] { - oauth2_ee::global_service().layer(Extension(Arc::clone(&sp_extension))) + oauth2_oss::global_service().layer(Extension(Arc::clone(&sp_extension))) } #[cfg(not(feature = "oauth2"))] @@ -721,7 +765,7 @@ pub async fn run_server( { #[cfg(all(feature = "enterprise", feature = "gcp_trigger"))] { - gcp_triggers_ee::gcp_push_route_handler() + gcp_triggers_oss::gcp_push_route_handler() } #[cfg(not(all(feature = "enterprise", feature = "gcp_trigger")))] { @@ -842,7 +886,7 @@ async fn ee_license() -> &'static str { #[cfg(feature = "enterprise")] async fn ee_license() -> String { - use windmill_common::ee::{LICENSE_KEY_ID, LICENSE_KEY_VALID}; + use windmill_common::ee_oss::{LICENSE_KEY_ID, LICENSE_KEY_VALID}; if *LICENSE_KEY_VALID.read().await { LICENSE_KEY_ID.read().await.clone() diff --git a/backend/windmill-api/src/mqtt_triggers.rs b/backend/windmill-api/src/mqtt_triggers.rs index 9f3fd19827..70b78ca913 100644 --- a/backend/windmill-api/src/mqtt_triggers.rs +++ b/backend/windmill-api/src/mqtt_triggers.rs @@ -39,7 +39,7 @@ use sql_builder::{bind::Bind, SqlBuilder}; use sqlx::{FromRow, Type}; use std::collections::HashMap; use std::time::Duration; -use windmill_audit::{audit_ee::audit_log, ActionKind}; +use windmill_audit::{audit_oss::audit_log, ActionKind}; use windmill_common::{ db::UserDB, error::{self, JsonResult}, diff --git a/backend/windmill-api/src/nats_triggers_ee.rs b/backend/windmill-api/src/nats_triggers_oss.rs similarity index 79% rename from backend/windmill-api/src/nats_triggers_ee.rs rename to backend/windmill-api/src/nats_triggers_oss.rs index 649d3a3837..28bf1e70e2 100644 --- a/backend/windmill-api/src/nats_triggers_ee.rs +++ b/backend/windmill-api/src/nats_triggers_oss.rs @@ -1,22 +1,34 @@ +#[cfg(feature = "private")] +#[allow(unused)] +pub use crate::nats_triggers_ee::*; + +#[cfg(not(feature = "private"))] use crate::db::DB; +#[cfg(not(feature = "private"))] use axum::Router; +#[cfg(not(feature = "private"))] use serde::{Deserialize, Serialize}; +#[cfg(not(feature = "private"))] #[derive(Serialize, Deserialize)] pub struct NatsResourceAuth {} +#[cfg(not(feature = "private"))] pub fn workspaced_service() -> Router { Router::new() } +#[cfg(not(feature = "private"))] pub fn start_nats_consumers(_db: DB, mut _killpill_rx: tokio::sync::broadcast::Receiver<()>) -> () { // implementation is not open source } #[derive(Serialize, Deserialize)] +#[cfg(not(feature = "private"))] pub enum NatsTriggerConfigConnection {} #[derive(Serialize, Clone)] +#[cfg(not(feature = "private"))] pub struct NatsTrigger { pub workspace_id: String, pub path: String, @@ -40,4 +52,4 @@ pub struct NatsTrigger { #[serde(skip_serializing_if = "Option::is_none")] pub error: Option, pub enabled: bool, -} \ No newline at end of file +} diff --git a/backend/windmill-api/src/oauth2_ee.rs b/backend/windmill-api/src/oauth2_oss.rs similarity index 76% rename from backend/windmill-api/src/oauth2_ee.rs rename to backend/windmill-api/src/oauth2_oss.rs index 49d2155cf3..04e0e1b202 100644 --- a/backend/windmill-api/src/oauth2_ee.rs +++ b/backend/windmill-api/src/oauth2_oss.rs @@ -1,3 +1,7 @@ +#[cfg(feature = "private")] +#[allow(unused)] +pub use crate::oauth2_ee::*; + /* * Author: Ruben Fiszel * Copyright: Windmill Labs, Inc 2022 @@ -6,39 +10,50 @@ * LICENSE-AGPL for a copy of the license. */ +#[cfg(not(feature = "private"))] use std::{collections::HashMap, fmt::Debug}; +#[cfg(not(feature = "private"))] use axum::{routing::get, Json, Router}; +#[cfg(not(feature = "private"))] use hmac::Mac; -#[cfg(feature = "oauth2")] +#[cfg(all(feature = "oauth2", not(feature = "private")))] use itertools::Itertools; -#[cfg(feature = "oauth2")] +#[cfg(all(feature = "oauth2", not(feature = "private")))] use oauth2::{Client as OClient, *}; +#[cfg(not(feature = "private"))] use serde::{Deserialize, Serialize}; +#[cfg(not(feature = "private"))] use sqlx::{Postgres, Transaction}; -#[cfg(feature = "oauth2")] +#[cfg(all(feature = "oauth2", not(feature = "private")))] use windmill_common::more_serde::maybe_number_opt; -#[cfg(feature = "oauth2")] +#[cfg(all(feature = "oauth2", not(feature = "private")))] use crate::OAUTH_CLIENTS; +#[cfg(not(feature = "private"))] use windmill_common::error; +#[cfg(not(feature = "private"))] use windmill_common::oauth2::*; +#[cfg(not(feature = "private"))] use crate::db::DB; +#[cfg(not(feature = "private"))] use std::str; +#[cfg(not(feature = "private"))] pub fn global_service() -> Router { Router::new() .route("/list_logins", get(list_logins)) .route("/list_connects", get(list_connects)) } +#[cfg(not(feature = "private"))] pub fn workspaced_service() -> Router { Router::new() } -#[cfg(feature = "oauth2")] +#[cfg(all(feature = "oauth2", not(feature = "private")))] #[derive(Debug, Clone)] pub struct ClientWithScopes { _client: OClient, @@ -48,9 +63,10 @@ pub struct ClientWithScopes { _allowed_domains: Option>, _userinfo_url: Option, } -#[cfg(feature = "oauth2")] +#[cfg(all(feature = "oauth2", not(feature = "private")))] pub type BasicClientsMap = HashMap; +#[cfg(not(feature = "private"))] #[derive(Clone, Debug, Serialize, Deserialize)] pub struct OAuthConfig { auth_url: String, @@ -62,6 +78,7 @@ pub struct OAuthConfig { req_body_auth: Option, } +#[cfg(not(feature = "private"))] #[derive(Clone, Debug, Serialize, Deserialize)] pub struct OAuthClient { id: String, @@ -71,7 +88,7 @@ pub struct OAuthClient { login_config: Option, } -#[cfg(feature = "oauth2")] +#[cfg(all(feature = "oauth2", not(feature = "private")))] #[derive(Debug)] pub struct AllClients { pub logins: BasicClientsMap, @@ -79,7 +96,7 @@ pub struct AllClients { pub slack: Option, } -#[cfg(feature = "oauth2")] +#[cfg(all(feature = "oauth2", not(feature = "private")))] pub async fn build_oauth_clients( _base_url: &str, _oauths_from_config: Option>, @@ -93,7 +110,7 @@ pub async fn build_oauth_clients( }); } -#[cfg(feature = "oauth2")] +#[cfg(all(feature = "oauth2", not(feature = "private")))] #[derive(Clone, Debug, Deserialize, Serialize)] pub struct TokenResponse { access_token: AccessToken, @@ -107,17 +124,20 @@ pub struct TokenResponse { scope: Option>, } +#[cfg(not(feature = "private"))] #[derive(Serialize)] struct Logins { oauth: Vec, saml: Option, } +#[cfg(not(feature = "private"))] async fn list_logins() -> error::JsonResult { // Implementation is not open source return Ok(Json(Logins { oauth: vec![], saml: None })); } -#[cfg(feature = "oauth2")] +#[allow(unused)] +#[cfg(all(feature = "oauth2", not(feature = "private")))] async fn list_connects() -> error::JsonResult> { Ok(Json( (&OAUTH_CLIENTS.read().await.connects) @@ -127,12 +147,14 @@ async fn list_connects() -> error::JsonResult> { )) } -#[cfg(not(feature = "oauth2"))] -async fn list_connects() -> error::JsonResult> { +#[allow(unused)] +#[cfg(not(all(feature = "oauth2", not(feature = "private"))))] +async fn list_connects() -> windmill_common::error::JsonResult> { // Implementation is not open source - return Ok(Json(vec![])); + return Ok(axum::Json(vec![])); } +#[cfg(not(feature = "private"))] pub async fn _refresh_token<'c>( _tx: Transaction<'c, Postgres>, _path: &str, @@ -146,6 +168,7 @@ pub async fn _refresh_token<'c>( )) } +#[cfg(not(feature = "private"))] pub async fn check_nb_of_user(db: &DB) -> error::Result<()> { let nb_users_sso = sqlx::query_scalar!("SELECT COUNT(*) FROM password WHERE login_type != 'password'",) @@ -171,10 +194,11 @@ pub async fn check_nb_of_user(db: &DB) -> error::Result<()> { } #[derive(Clone, Debug)] +#[cfg(not(feature = "private"))] pub struct SlackVerifier { _mac: HmacSha256, } - +#[cfg(not(feature = "private"))] impl SlackVerifier { pub fn new>(secret: S) -> anyhow::Result { HmacSha256::new_from_slice(secret.as_ref()) diff --git a/backend/windmill-api/src/oidc_ee.rs b/backend/windmill-api/src/oidc_oss.rs similarity index 68% rename from backend/windmill-api/src/oidc_ee.rs rename to backend/windmill-api/src/oidc_oss.rs index 248b990f54..4042c2a53b 100644 --- a/backend/windmill-api/src/oidc_ee.rs +++ b/backend/windmill-api/src/oidc_oss.rs @@ -1,3 +1,7 @@ +#[cfg(feature = "private")] +#[allow(unused)] +pub use crate::oidc_ee::*; + /* * Author: Ruben Fiszel * Copyright: Windmill Labs, Inc 2023 @@ -6,12 +10,15 @@ * LICENSE-AGPL for a copy of the license. */ +#[cfg(not(feature = "private"))] use axum::Router; +#[cfg(not(feature = "private"))] pub fn global_service() -> Router { Router::new() } +#[cfg(not(feature = "private"))] pub fn workspaced_service() -> Router { Router::new() } diff --git a/backend/windmill-api/src/postgres_triggers/handler.rs b/backend/windmill-api/src/postgres_triggers/handler.rs index 933582670c..73e35c9e2f 100644 --- a/backend/windmill-api/src/postgres_triggers/handler.rs +++ b/backend/windmill-api/src/postgres_triggers/handler.rs @@ -6,7 +6,6 @@ use std::collections::{ use crate::{ db::{ApiAuthed, DB}, postgres_triggers::mapper::{Mapper, MappingInfo}, - resources::try_get_resource_from_db_as, }; use axum::{ extract::{Path, Query}, @@ -16,25 +15,25 @@ use http::StatusCode; use itertools::Itertools; use pg_escape::{quote_identifier, quote_literal}; use quick_cache::sync::Cache; -use rust_postgres::types::Type; +use rust_postgres::{types::Type, Client}; use serde::{Deserialize, Deserializer, Serialize}; use sql_builder::{bind::Bind, SqlBuilder}; -use sqlx::{postgres::types::Oid, FromRow, PgConnection}; -use windmill_audit::{audit_ee::audit_log, ActionKind}; -use windmill_common::error::Error; +use sqlx::FromRow; +use windmill_audit::{audit_oss::audit_log, ActionKind}; use windmill_common::{ db::UserDB, - error::{self, JsonResult, Result}, - utils::{not_found_if_none, paginate, Pagination, StripPath, empty_as_none}, + error::{self, to_anyhow, Error, JsonResult, Result}, + utils::{empty_as_none, not_found_if_none, paginate, Pagination, StripPath}, worker::CLOUD_HOSTED, }; use windmill_git_sync::{handle_deployment_metadata, DeployedObject}; use super::{ - create_logical_replication_slot_query, create_publication_query, drop_publication_query, - generate_random_string, get_database_connection, get_raw_postgres_connection, - ERROR_PUBLICATION_NAME_NOT_EXISTS, ERROR_REPLICATION_SLOT_NOT_EXISTS, + check_if_valid_publication_for_postgres_version, create_logical_replication_slot, + create_pg_publication, drop_publication, generate_random_string, get_default_pg_connection, + ERROR_PUBLICATION_NAME_NOT_EXISTS, }; +use anyhow::anyhow; use lazy_static::lazy_static; #[derive(FromRow, Serialize, Deserialize, Debug)] @@ -53,15 +52,17 @@ pub struct Postgres { #[derive(Debug, Clone, FromRow, Serialize, Deserialize)] pub struct TableToTrack { pub table_name: String, + #[serde(default, deserialize_with = "empty_as_none")] pub where_clause: Option, - pub columns_name: Vec, + #[serde(default, deserialize_with = "empty_as_none")] + pub columns_name: Option>, } impl TableToTrack { fn new( table_name: String, where_clause: Option, - columns_name: Vec, + columns_name: Option>, ) -> TableToTrack { TableToTrack { table_name, where_clause, columns_name } } @@ -99,7 +100,6 @@ pub struct EditPostgresTrigger { } #[derive(Deserialize, Serialize, Debug)] - pub struct NewPostgresTrigger { path: String, script_path: String, @@ -124,7 +124,7 @@ pub async fn test_postgres_connection( Json(test_postgres): Json, ) -> Result<()> { let connect_f = async { - get_database_connection( + get_default_pg_connection( authed, Some(user_db), &db, @@ -189,7 +189,7 @@ where )); } - if !track_specific_columns_in_table && !table_to_track.columns_name.is_empty() { + if !track_specific_columns_in_table && table_to_track.columns_name.is_some() { track_specific_columns_in_table = true; } } @@ -267,39 +267,18 @@ impl PostgresPublicationReplication { } } -async fn check_if_publication_exist( - connection: &mut PgConnection, - publication_name: &str, -) -> Result<()> { - sqlx::query!( - "SELECT pubname FROM pg_publication WHERE pubname = $1", - publication_name - ) - .fetch_one(connection) - .await - .map_err(|err| match err { - sqlx::Error::RowNotFound => { - Error::BadRequest(ERROR_PUBLICATION_NAME_NOT_EXISTS.to_string()) - } - err => Error::SqlErr { error: err, location: "pg_trigger".to_string() }, - })?; - Ok(()) -} - async fn check_if_logical_replication_slot_exist( - connection: &mut PgConnection, + pg_connection: &mut Client, replication_slot_name: &str, -) -> Result<()> { - sqlx::query!( - "SELECT slot_name FROM pg_replication_slots where slot_name = $1", - &replication_slot_name - ) - .fetch_one(connection) - .await - .map_err(|err| match err { - _ => Error::BadRequest(ERROR_REPLICATION_SLOT_NOT_EXISTS.to_string()), - })?; - Ok(()) +) -> Result { + let row = pg_connection + .query_opt( + "SELECT slot_name FROM pg_replication_slots WHERE slot_name = $1", + &[&replication_slot_name], + ) + .await + .map_err(to_anyhow)?; + Ok(row.is_some()) } async fn create_custom_slot_and_publication_inner( @@ -310,33 +289,31 @@ async fn create_custom_slot_and_publication_inner( w_id: &str, publication: &PublicationData, ) -> Result { - let publication_name = format!("windmill_trigger_{}", generate_random_string()); - let replication_slot_name = publication_name.clone(); - - let query = create_publication_query( - &publication_name, - publication.table_to_track.as_deref(), - &publication - .transaction_to_track - .iter() - .map(AsRef::as_ref) - .collect_vec(), - ); - - let mut connection = get_database_connection( + let mut pg_connection = get_default_pg_connection( authed.clone(), - Some(user_db.clone()), + Some(user_db), &db, &postgres_resource_path, &w_id, ) + .await + .map_err(to_anyhow)?; + + let tx = pg_connection.transaction().await.map_err(to_anyhow)?; + + let publication_name = format!("windmill_trigger_{}", generate_random_string()); + let replication_slot_name = publication_name.clone(); + + create_logical_replication_slot(tx.client(), &replication_slot_name).await?; + create_pg_publication( + &tx.client(), + &publication_name, + publication.table_to_track.as_deref(), + &publication.transaction_to_track, + ) .await?; - sqlx::query(&query).execute(&mut connection).await?; - - let query = create_logical_replication_slot_query(&replication_slot_name); - - sqlx::query(&query).execute(&mut connection).await?; + tx.commit().await.map_err(to_anyhow)?; Ok(PostgresPublicationReplication::new( publication_name, @@ -344,6 +321,38 @@ async fn create_custom_slot_and_publication_inner( )) } +pub async fn get_postgres_version_internal(pg_connection: &Client) -> Result { + let row = pg_connection + .query_one("SHOW server_version;", &[]) + .await + .map_err(to_anyhow)?; + + let postgres_version: String = row.get(0); + + Ok(postgres_version) +} + +pub async fn get_postgres_version( + authed: ApiAuthed, + Extension(db): Extension, + Extension(user_db): Extension, + Path((w_id, postgres_resource_path)): Path<(String, String)>, +) -> Result { + let pg_connection = get_default_pg_connection( + authed.clone(), + Some(user_db), + &db, + &postgres_resource_path, + &w_id, + ) + .await + .map_err(to_anyhow)?; + + let postgres_version = get_postgres_version_internal(&pg_connection).await?; + + Ok(postgres_version) +} + pub async fn create_postgres_trigger( authed: ApiAuthed, Extension(user_db): Extension, @@ -377,6 +386,7 @@ pub async fn create_postgres_trigger( if publication.is_none() { return Err(Error::BadRequest("publication must be set".to_string())); } + let PostgresPublicationReplication { publication_name, replication_slot_name } = create_custom_slot_and_publication_inner( authed.clone(), @@ -577,7 +587,7 @@ impl PublicationData { } } -#[derive(Debug, Serialize)] +#[derive(FromRow, Debug, Serialize)] pub struct SlotList { slot_name: Option, active: Option, @@ -589,30 +599,37 @@ pub async fn list_slot_name( Extension(db): Extension, Path((w_id, postgres_resource_path)): Path<(String, String)>, ) -> Result>> { - let mut connection = get_database_connection( + let pg_connection: Client = get_default_pg_connection( authed.clone(), Some(user_db.clone()), &db, &postgres_resource_path, &w_id, ) - .await?; + .await + .map_err(to_anyhow)?; - let slots = sqlx::query_as!( - SlotList, - r#" - SELECT - slot_name, - active - FROM - pg_replication_slots - WHERE - plugin = 'pgoutput' AND - slot_type = 'logical'; - "# - ) - .fetch_all(&mut connection) - .await?; + let rows = pg_connection + .query( + r#" + SELECT + slot_name, + active + FROM + pg_replication_slots + WHERE + plugin = 'pgoutput' AND + slot_type = 'logical'; + "#, + &[], + ) + .await + .map_err(to_anyhow)?; + + let slots = rows + .into_iter() + .map(|row| SlotList { slot_name: row.get("slot_name"), active: row.get("active") }) + .collect(); Ok(Json(slots)) } @@ -629,22 +646,54 @@ pub async fn create_slot( Path((w_id, postgres_resource_path)): Path<(String, String)>, Json(Slot { name }): Json, ) -> Result { - let mut connection = get_database_connection( + let pg_connection = get_default_pg_connection( authed.clone(), Some(user_db.clone()), &db, &postgres_resource_path, &w_id, ) - .await?; + .await + .map_err(to_anyhow)?; - let query = create_logical_replication_slot_query(&name); - - sqlx::query(&query).execute(&mut connection).await?; + create_logical_replication_slot(&pg_connection, &name).await?; Ok(format!("Replication slot {} created!", name)) } +pub async fn drop_logical_replication_slot(pg_connection: &Client, slot_name: &str) -> Result<()> { + let row = pg_connection + .query_opt( + r#" + SELECT + active_pid + FROM + pg_replication_slots + WHERE + slot_name = $1 + "#, + &[&slot_name], + ) + .await + .map_err(to_anyhow)?; + + let active_pid = row.map(|r| r.get::<_, Option>(0)).flatten(); + + if let Some(pid) = active_pid { + pg_connection + .execute("SELECT pg_terminate_backend($1)", &[&pid]) + .await + .map_err(to_anyhow)?; + } + + pg_connection + .execute("SELECT pg_drop_replication_slot($1)", &[&slot_name]) + .await + .map_err(to_anyhow)?; + + Ok(()) +} + pub async fn drop_slot_name( authed: ApiAuthed, Extension(user_db): Extension, @@ -652,48 +701,17 @@ pub async fn drop_slot_name( Path((w_id, postgres_resource_path)): Path<(String, String)>, Json(Slot { name }): Json, ) -> Result { - let database = try_get_resource_from_db_as::( - authed, - Some(user_db), - &db, - &postgres_resource_path, - &w_id, - ) - .await?; + let pg_connection = + get_default_pg_connection(authed, Some(user_db), &db, &postgres_resource_path, &w_id) + .await + .map_err(to_anyhow)?; - let mut connection = get_raw_postgres_connection(&database).await?; - - let active_pid = sqlx::query_scalar!( - r#"SELECT - active_pid - FROM - pg_replication_slots - WHERE - slot_name = $1 - "#, - &name - ) - .fetch_optional(&mut connection) - .await? - .flatten(); - - if let Some(pid) = active_pid { - sqlx::query("SELECT pg_terminate_backend($1)") - .bind(pid) - .execute(&mut connection) - .await?; - } - sqlx::query("SELECT pg_drop_replication_slot($1)") - .bind(&name) - .execute(&mut connection) - .await?; + drop_logical_replication_slot(&pg_connection, &name) + .await + .map_err(to_anyhow)?; Ok(format!("Replication slot {} deleted!", name)) } -#[derive(Debug, Serialize)] -struct PublicationName { - publication_name: String, -} pub async fn list_database_publication( authed: ApiAuthed, @@ -701,25 +719,27 @@ pub async fn list_database_publication( Extension(db): Extension, Path((w_id, postgres_resource_path)): Path<(String, String)>, ) -> Result>> { - let mut connection = get_database_connection( + let pg_connection = get_default_pg_connection( authed.clone(), Some(user_db.clone()), &db, &postgres_resource_path, &w_id, ) - .await?; + .await + .map_err(to_anyhow)?; - let publication_names = sqlx::query_as!( - PublicationName, - "SELECT pubname AS publication_name FROM pg_publication;" - ) - .fetch_all(&mut connection) - .await?; + let rows = pg_connection + .query( + "SELECT pubname AS publication_name FROM pg_publication;", + &[], + ) + .await + .map_err(to_anyhow)?; - let publications = publication_names - .iter() - .map(|publication| publication.publication_name.to_owned()) + let publications = rows + .into_iter() + .map(|row| row.get::<_, String>("publication_name")) .collect_vec(); Ok(Json(publications)) @@ -731,21 +751,22 @@ pub async fn get_publication_info( Extension(db): Extension, Path((w_id, publication_name, postgres_resource_path)): Path<(String, String, String)>, ) -> Result> { - let mut connection = get_database_connection( + let mut pg_connection = get_default_pg_connection( authed.clone(), Some(user_db.clone()), &db, &postgres_resource_path, &w_id, ) - .await?; + .await + .map_err(to_anyhow)?; let publication_data = - get_publication_scope_and_transaction(&mut connection, &publication_name).await; + get_publication_scope_and_transaction(&mut pg_connection, &publication_name).await; let (all_table, transaction_to_track) = match publication_data { - Ok(pub_data) => pub_data, - Err(Error::SqlErr { error: sqlx::Error::RowNotFound, .. }) => { + Ok(Some(pub_data)) => pub_data, + Ok(None) => { return Err(Error::NotFound( ERROR_PUBLICATION_NAME_NOT_EXISTS.to_string(), )) @@ -754,7 +775,7 @@ pub async fn get_publication_info( }; let table_to_track = if !all_table { - Some(get_tracked_relations(&mut connection, &publication_name).await?) + Some(get_tracked_relations(&mut pg_connection, &publication_name).await?) } else { None }; @@ -771,24 +792,29 @@ pub async fn create_publication( Path((w_id, publication_name, postgres_resource_path)): Path<(String, String, String)>, Json(publication_data): Json, ) -> Result { - let PublicationData { table_to_track, transaction_to_track } = publication_data; - - let mut connection = get_database_connection( + let mut pg_connection = get_default_pg_connection( authed.clone(), Some(user_db.clone()), &db, &postgres_resource_path, &w_id, ) - .await?; + .await + .map_err(to_anyhow)?; - let query = create_publication_query( + let PublicationData { table_to_track, transaction_to_track } = publication_data; + + let tx = pg_connection.transaction().await.map_err(to_anyhow)?; + + create_pg_publication( + tx.client(), &publication_name, table_to_track.as_deref(), - &transaction_to_track.iter().map(AsRef::as_ref).collect_vec(), - ); + &transaction_to_track, + ) + .await?; - sqlx::query(&query).execute(&mut connection).await?; + tx.commit().await.map_err(to_anyhow)?; Ok(format!( "Publication {} successfully created!", @@ -802,18 +828,17 @@ pub async fn delete_publication( Extension(db): Extension, Path((w_id, publication_name, postgres_resource_path)): Path<(String, String, String)>, ) -> Result { - let mut connection = get_database_connection( + let mut pg_connection = get_default_pg_connection( authed.clone(), Some(user_db.clone()), &db, &postgres_resource_path, &w_id, ) - .await?; + .await + .map_err(to_anyhow)?; - let query = drop_publication_query(&publication_name); - - sqlx::query(&query).execute(&mut connection).await?; + drop_publication(&mut pg_connection, &publication_name).await?; Ok(format!( "Publication {} successfully deleted!", @@ -821,57 +846,68 @@ pub async fn delete_publication( )) } -pub fn get_update_publication_query( +pub async fn update_pg_publication( + pg_connection: &Client, publication_name: &str, PublicationData { table_to_track, transaction_to_track }: PublicationData, - all_table: bool, -) -> Vec { - let quoted_publication_name = quote_identifier(&publication_name); - + all_table: Option, +) -> Result<()> { + let quoted_publication_name = quote_identifier(publication_name); let transaction_to_track_as_str = transaction_to_track.iter().join(","); - let mut queries = Vec::with_capacity(2); match table_to_track { Some(ref relations) if !relations.is_empty() => { - if all_table { - queries.push(drop_publication_query(&publication_name)); - queries.push(create_publication_query( - &publication_name, + // If all_table is None, the publication does not exist yet + if all_table.unwrap_or(true) { + if all_table.is_some_and(|all_table| all_table) { + drop_publication(pg_connection, publication_name) + .await + .map_err(to_anyhow)?; + } + create_pg_publication( + pg_connection, + publication_name, table_to_track.as_deref(), - &transaction_to_track.iter().map(AsRef::as_ref).collect_vec(), - )); + &transaction_to_track, + ) + .await + .map_err(to_anyhow)?; } else { - let mut query = String::from(""); + let pg_14 = check_if_valid_publication_for_postgres_version( + pg_connection, + table_to_track.as_deref(), + ) + .await + .map_err(to_anyhow)?; + + let mut query = format!("ALTER PUBLICATION {} SET ", quoted_publication_name); + let mut first = true; - query.push_str("ALTER PUBLICATION "); - query.push_str("ed_publication_name); - query.push_str(" SET"); for (i, schema) in relations.iter().enumerate() { if schema.table_to_track.is_empty() { - query.push_str(" TABLES IN SCHEMA "); - let quoted_schema = quote_identifier(&schema.schema_name); - query.push_str("ed_schema); + query.push_str("TABLES IN SCHEMA "); + query.push_str("e_identifier(&schema.schema_name)); } else { - query.push_str(" TABLE ONLY "); + if pg_14 && first { + query.push_str("TABLE ONLY "); + first = false; + } else if !pg_14 { + query.push_str("TABLE ONLY "); + } + for (j, table) in schema.table_to_track.iter().enumerate() { let table_name = quote_identifier(&table.table_name); let schema_name = quote_identifier(&schema.schema_name); - let full_name = format!("{}.{}", &schema_name, &table_name); + let full_name = format!("{}.{}", schema_name, table_name); query.push_str(&full_name); - if !table.columns_name.is_empty() { - query.push_str(" ("); - let columns = table - .columns_name - .iter() - .map(|column| quote_identifier(column)) - .join(", "); - query.push_str(&columns); - query.push_str(") "); + + if let Some(columns) = table.columns_name.as_ref() { + let cols = + columns.iter().map(|col| quote_identifier(col)).join(", "); + query.push_str(&format!(" ({})", cols)); } if let Some(where_clause) = &table.where_clause { - query.push_str(" WHERE ("); - query.push_str(where_clause); - query.push(')'); + query.push_str(&format!(" WHERE ({})", where_clause)); } if j + 1 != schema.table_to_track.len() { @@ -879,39 +915,43 @@ pub fn get_update_publication_query( } } } - if i < relations.len() - 1 { - query.push(','); + + if i + 1 != relations.len() { + query.push_str(", "); } } - query.push(';'); - queries.push(query); + pg_connection + .execute(&query, &[]) + .await + .map_err(to_anyhow)?; - let mut query = String::new(); - - query.push_str("ALTER PUBLICATION "); - query.push_str("ed_publication_name); - query.push_str(&format!( - " SET (publish = '{}');", - transaction_to_track_as_str - )); - queries.push(query); + let publish_query = format!( + "ALTER PUBLICATION {} SET (publish = '{}');", + quoted_publication_name, transaction_to_track_as_str + ); + pg_connection + .execute(&publish_query, &[]) + .await + .map_err(to_anyhow)?; } } _ => { - queries.push(drop_publication_query(&publication_name)); - let to_execute = format!( - r#" - CREATE - PUBLICATION {} FOR ALL TABLES WITH (publish = '{}'); - "#, + drop_publication(pg_connection, publication_name) + .await + .map_err(to_anyhow)?; + let create_all_query = format!( + "CREATE PUBLICATION {} FOR ALL TABLES WITH (publish = '{}');", quoted_publication_name, transaction_to_track_as_str ); - queries.push(to_execute); + pg_connection + .execute(&create_all_query, &[]) + .await + .map_err(to_anyhow)?; } - }; + } - queries + Ok(()) } pub async fn alter_publication( @@ -921,25 +961,32 @@ pub async fn alter_publication( Path((w_id, publication_name, postgres_resource_path)): Path<(String, String, String)>, Json(publication_data): Json, ) -> Result { - let mut connection = get_database_connection( + let mut pg_connection = get_default_pg_connection( authed.clone(), Some(user_db.clone()), &db, &postgres_resource_path, &w_id, ) - .await?; + .await + .map_err(to_anyhow)?; - check_if_publication_exist(&mut connection, &publication_name).await?; + let tx = pg_connection.transaction().await.map_err(to_anyhow)?; - let (all_table, _) = - get_publication_scope_and_transaction(&mut connection, &publication_name).await?; + let publication = get_publication_scope_and_transaction(tx.client(), &publication_name) + .await + .map_err(to_anyhow)?; - let queries = get_update_publication_query(&publication_name, publication_data, all_table); + update_pg_publication( + tx.client(), + &publication_name, + publication_data, + publication.map(|publication| publication.0), + ) + .await + .map_err(to_anyhow)?; - for query in queries { - sqlx::query(&query).execute(&mut connection).await?; - } + tx.commit().await.map_err(to_anyhow)?; Ok(format!( "Publication {} updated with success", @@ -947,104 +994,126 @@ pub async fn alter_publication( )) } -async fn get_publication_scope_and_transaction( - connection: &mut PgConnection, +pub async fn get_publication_scope_and_transaction( + pg_connection: &Client, publication_name: &str, -) -> std::result::Result<(bool, Vec), Error> { - #[derive(Debug, Deserialize, FromRow)] - struct PublicationTransaction { - all_table: bool, - insert: bool, - update: bool, - delete: bool, - } +) -> Result)>> { + let row_opt = pg_connection + .query_opt( + r#" + SELECT + puballtables AS all_table, + pubinsert AS insert, + pubupdate AS update, + pubdelete AS delete + FROM + pg_publication + WHERE + pubname = $1 + "#, + &[&publication_name], + ) + .await + .map_err(to_anyhow)?; - let transaction = sqlx::query_as!( - PublicationTransaction, - r#" - SELECT - puballtables AS all_table, - pubinsert AS insert, - pubupdate AS update, - pubdelete AS delete - FROM - pg_publication - WHERE - pubname = $1 - "#, - publication_name - ) - .fetch_one(&mut *connection) - .await?; + let row = match row_opt { + Some(r) => r, + None => return Ok(None), + }; + + let all_table: bool = row.get("all_table"); + let pub_insert: bool = row.get("insert"); + let pub_update: bool = row.get("update"); + let pub_delete: bool = row.get("delete"); let mut transaction_to_track = Vec::with_capacity(3); - - if transaction.insert { + if pub_insert { transaction_to_track.push("insert".to_string()); } - if transaction.update { + if pub_update { transaction_to_track.push("update".to_string()); } - if transaction.delete { + if pub_delete { transaction_to_track.push("delete".to_string()); } - Ok((transaction.all_table, transaction_to_track)) + Ok(Some((all_table, transaction_to_track))) } -async fn get_tracked_relations( - connection: &mut PgConnection, +pub async fn get_tracked_relations( + pg_connection: &Client, publication_name: &str, ) -> Result> { - #[derive(Debug, Deserialize, FromRow)] - struct PublicationData { - schema_name: Option, - table_name: Option, - columns: Option>, - where_clause: Option, - } + let pg_version = get_postgres_version_internal(pg_connection).await?; - let publications = sqlx::query_as!( - PublicationData, + let query = if pg_version.starts_with("14") { r#" - SELECT + SELECT schemaname AS schema_name, tablename AS table_name, - CASE - WHEN array_length(attnames, 1) = (SELECT COUNT(*) FROM information_schema.columns WHERE table_schema = pg_publication_tables.schemaname AND table_name = pg_publication_tables.tablename) - THEN NULL - ELSE attnames - END AS columns, + NULL::text[] AS columns, + NULL::text AS where_clause + FROM + pg_publication_tables + WHERE + pubname = $1; + "# + } else { + r#" + SELECT + schemaname AS schema_name, + tablename AS table_name, + attnames AS columns, rowfilter AS where_clause - FROM - pg_publication_tables - WHERE - pubname = $1; - "#, - publication_name - ) - .fetch_all(&mut *connection) - .await?; + FROM + pg_publication_tables + WHERE + pubname = $1; + "# + }; + + let rows = pg_connection + .query(query, &[&publication_name]) + .await + .map_err(to_anyhow)?; let mut table_to_track: HashMap = HashMap::new(); - for publication in publications { - let schema_name = publication.schema_name.unwrap(); + for row in rows { + let schema_name: Option = row.get("schema_name"); + let table_name: Option = row.get("table_name"); + let columns: Option> = row.get("columns"); + let where_clause: Option = row.get("where_clause"); + + let schema_name = schema_name.ok_or_else::( || { + anyhow!( + "Unexpected NULL `schema_name` in publication entry (pubname: `{}`). This should never happen unless PostgreSQL internals are corrupted.", + publication_name, + ).into() + } + )?; + + let table_name = table_name.ok_or_else::(|| { + anyhow!( + "Unexpected NULL `table_name` for schema `{}` in publication `{}`. This should never happen unless PostgreSQL internals are corrupted.", + schema_name, + publication_name, + ).into() + })?; + let entry = table_to_track.entry(schema_name.clone()); - let table_to_track = TableToTrack::new( - publication.table_name.unwrap(), - publication.where_clause, - publication.columns.unwrap_or_default(), - ); + let table_to_track = TableToTrack::new(table_name, where_clause, columns); + match entry { - Occupied(mut occuped) => { + std::collections::hash_map::Entry::Occupied(mut occuped) => { occuped.get_mut().add_new_table(table_to_track); } - Vacant(vacant) => { + std::collections::hash_map::Entry::Vacant(vacant) => { vacant.insert(Relations::new(schema_name, vec![table_to_track])); } } } + Ok(table_to_track.into_values().collect_vec()) } @@ -1111,27 +1180,49 @@ pub async fn update_postgres_trigger( publication, } = postgres_trigger; - let mut connection = get_database_connection( + let mut pg_connection = get_default_pg_connection( authed.clone(), Some(user_db.clone()), &db, &postgres_resource_path, &w_id, ) - .await?; + .await + .map_err(to_anyhow)?; - check_if_logical_replication_slot_exist(&mut connection, &replication_slot_name).await?; + let exists = + check_if_logical_replication_slot_exist(&mut pg_connection, &replication_slot_name).await?; + + let tx = pg_connection.transaction().await.map_err(to_anyhow)?; + + if !exists { + tracing::debug!( + "Logical replication slot named: {} does not exists creating it...", + &replication_slot_name + ); + create_logical_replication_slot(tx.client(), &replication_slot_name) + .await + .map_err(to_anyhow)?; + } if let Some(publication) = publication { - check_if_publication_exist(&mut connection, &publication_name).await?; - let (all_table, _) = - get_publication_scope_and_transaction(&mut connection, &publication_name).await?; + let publication_data = + get_publication_scope_and_transaction(tx.client(), &publication_name) + .await + .map_err(to_anyhow)?; - let queries = get_update_publication_query(&publication_name, publication, all_table); - for query in queries { - sqlx::query(&query).execute(&mut connection).await?; - } + update_pg_publication( + tx.client(), + &publication_name, + publication, + publication_data.map(|publication| publication.0), + ) + .await + .map_err(to_anyhow)?; } + + tx.commit().await.map_err(to_anyhow)?; + let mut tx = user_db.begin(&authed).await?; sqlx::query!( @@ -1350,52 +1441,40 @@ pub async fn create_template_script( Json(template_script): Json, ) -> Result { let TemplateScript { postgres_resource_path, relations, language } = template_script; - if relations.is_none() { - return Err(Error::BadRequest( - "You must at least choose schema to fetch table from".to_string(), - )); - } - let mut connection = get_database_connection( + let relations = match relations { + Some(r) => r, + None => return Err(anyhow!("You must at least choose schema to fetch table from").into()), + }; + + let pg_connection: Client = get_default_pg_connection( authed.clone(), Some(user_db.clone()), &db, &postgres_resource_path, &w_id, ) - .await?; + .await + .map_err(to_anyhow)?; - #[derive(Debug, FromRow, Deserialize)] - struct ColumnInfo { - table_schema: Option, - table_name: Option, - column_name: Option, - oid: Oid, - is_nullable: bool, - } - - let relations = relations.unwrap(); let mut schema_or_fully_qualified_name = Vec::with_capacity(relations.len()); - let mut columns_list = Vec::new(); + let mut columns_list = Vec::with_capacity(relations.len()); + for relation in relations { if !relation.table_to_track.is_empty() { - for table_to_track in relation.table_to_track { - let fully_qualified_name = - format!("{}.{}", &relation.schema_name, table_to_track.table_name); + for table in relation.table_to_track { + let fully_qualified_name = format!("{}.{}", relation.schema_name, table.table_name); schema_or_fully_qualified_name.push(quote_literal(&fully_qualified_name)); - - let columns = if !table_to_track.columns_name.is_empty() { - quote_literal(&table_to_track.columns_name.join(",")) - } else { - "''".to_string() - }; + let columns = table + .columns_name + .map(|c| quote_literal(&c.join(","))) + .unwrap_or_else(|| "''".to_string()); columns_list.push(columns); } - continue; + } else { + schema_or_fully_qualified_name.push(quote_literal(&relation.schema_name)); + columns_list.push("''".to_string()); } - - schema_or_fully_qualified_name.push(quote_literal(&relation.schema_name)); - columns_list.push(String::from("''")); } let tables_name = schema_or_fully_qualified_name.join(","); @@ -1415,8 +1494,7 @@ pub async fn create_template_script( WHEN tcm.column_list = '' THEN NULL ELSE string_to_array(tcm.column_list, ',') END AS columns - FROM - table_column_mapping tcm + FROM table_column_mapping tcm ) SELECT ns.nspname AS table_schema, @@ -1424,22 +1502,16 @@ pub async fn create_template_script( attr.attname AS column_name, attr.atttypid AS oid, attr.attnotnull AS is_nullable - FROM - pg_attribute attr - JOIN - pg_class cls - ON attr.attrelid = cls.oid - JOIN - pg_namespace ns - ON cls.relnamespace = ns.oid - JOIN - parsed_columns pc + FROM pg_attribute attr + JOIN pg_class cls ON attr.attrelid = cls.oid + JOIN pg_namespace ns ON cls.relnamespace = ns.oid + JOIN parsed_columns pc ON ns.nspname || '.' || cls.relname = pc.table_name OR ns.nspname = pc.table_name WHERE - attr.attnum > 0 -- Exclude system columns - AND NOT attr.attisdropped -- Exclude dropped columns - AND cls.relkind = 'r' -- Restrict to base tables + attr.attnum > 0 + AND NOT attr.attisdropped + AND cls.relkind = 'r' AND ( pc.columns IS NULL OR attr.attname = ANY(pc.columns) @@ -1448,55 +1520,52 @@ pub async fn create_template_script( tables_name, columns_list ); - let rows: Vec = sqlx::query_as(&query) - .fetch_all(&mut connection) - .await - .map_err(|e| error::Error::SqlErr { error: e, location: "pg_trigger".to_string() })?; + let rows = pg_connection.query(&query, &[]).await.map_err(to_anyhow)?; - let mut mapper: HashMap>> = HashMap::new(); + let mut schema_map: HashMap>> = HashMap::new(); + + #[derive(Debug)] + struct ColumnInfo { + table_schema: String, + table_name: String, + column_name: String, + oid: u32, + is_nullable: bool, + } for row in rows { - let ColumnInfo { table_schema, table_name, column_name, oid, is_nullable } = row; - - let entry = mapper.entry(table_schema.unwrap()); + let info = ColumnInfo { + table_schema: row.get("table_schema"), + table_name: row.get("table_name"), + column_name: row.get("column_name"), + oid: row.get::<_, u32>("oid"), + is_nullable: row.get::<_, bool>("is_nullable"), + }; let mapped_info = - MappingInfo::new(column_name.unwrap(), Type::from_oid(oid.0), is_nullable); + MappingInfo::new(info.column_name, Type::from_oid(info.oid), info.is_nullable); - match entry { - Occupied(mut occupied) => { - let entry = occupied.get_mut().entry(table_name.unwrap()); - match entry { - Occupied(mut occuped) => { - let mapping_info = occuped.get_mut(); - mapping_info.push(mapped_info); - } - Vacant(vacant) => { - let mut mapping_info = Vec::with_capacity(10); - mapping_info.push(mapped_info); - vacant.insert(mapping_info); - } + match schema_map.entry(info.table_schema) { + Occupied(mut schema_entry) => match schema_entry.get_mut().entry(info.table_name) { + Occupied(mut table_entry) => { + table_entry.get_mut().push(mapped_info); } - } - Vacant(vacant) => { - let mut mapping_info = Vec::with_capacity(10); - mapping_info.push(mapped_info); - vacant.insert(HashMap::from([(table_name.unwrap(), mapping_info)])); + Vacant(v) => { + v.insert(vec![mapped_info]); + } + }, + Vacant(schema_vacant) => { + let mut table_map = HashMap::new(); + table_map.insert(info.table_name, vec![mapped_info]); + schema_vacant.insert(table_map); } } } - let mapper = Mapper::new(mapper, language); - - let create_template_id = |w_id: &str| -> String { - let uuid = uuid::Uuid::new_v4().to_string(); - let id = format!("{}-{}", &w_id, &uuid); - - id - }; - + let mapper = Mapper::new(schema_map, language); let template = mapper.get_template(); - let id = create_template_id(&w_id); + + let id = format!("{}-{}", w_id, uuid::Uuid::new_v4()); TEMPLATE.insert(id.clone(), template); @@ -1509,24 +1578,24 @@ pub async fn is_database_in_logical_level( Extension(db): Extension, Path((w_id, postgres_resource_path)): Path<(String, String)>, ) -> error::JsonResult { - let mut connection = get_database_connection( + let pg_connection = get_default_pg_connection( authed.clone(), Some(user_db.clone()), &db, &postgres_resource_path, &w_id, ) - .await?; + .await + .map_err(to_anyhow)?; - let wal_level = sqlx::query_scalar!("SHOW WAL_LEVEL;") - .fetch_optional(&mut connection) - .await? - .flatten(); + let row_opt = pg_connection + .query_opt("SHOW wal_level;", &[]) + .await + .map_err(to_anyhow)?; - let is_logical = match wal_level.as_deref() { - Some("logical") => true, - _ => false, - }; + let wal_level: Option = row_opt.map(|row| row.get(0)); + + let is_logical = matches!(wal_level.as_deref(), Some("logical")); Ok(Json(is_logical)) } diff --git a/backend/windmill-api/src/postgres_triggers/mod.rs b/backend/windmill-api/src/postgres_triggers/mod.rs index d557ca67cc..096f0c78d0 100644 --- a/backend/windmill-api/src/postgres_triggers/mod.rs +++ b/backend/windmill-api/src/postgres_triggers/mod.rs @@ -7,15 +7,13 @@ use crate::{ }; use chrono::Utc; use itertools::Itertools; -use pg_escape::{quote_identifier, quote_literal}; +use native_tls::{Certificate, TlsConnector}; +use pg_escape::quote_identifier; use rand::Rng; +use rust_postgres::{config::SslMode, Client, Config, NoTls}; +use rust_postgres_native_tls::MakeTlsConnector; use serde_json::value::RawValue; -use sqlx::{ - postgres::{PgConnectOptions, PgSslMode}, - Connection, PgConnection, -}; use std::collections::HashMap; -use std::str::FromStr; use axum::{ routing::{delete, get, post}, @@ -25,12 +23,17 @@ pub use handler::PostgresTrigger; use handler::{ alter_publication, create_postgres_trigger, create_publication, create_slot, create_template_script, delete_postgres_trigger, delete_publication, drop_slot_name, - exists_postgres_trigger, get_postgres_trigger, get_publication_info, get_template_script, + exists_postgres_trigger, get_postgres_trigger, get_postgres_version, + get_postgres_version_internal, get_publication_info, get_template_script, is_database_in_logical_level, list_database_publication, list_postgres_triggers, list_slot_name, set_enabled, test_postgres_connection, update_postgres_trigger, Postgres, Relations, }; -use windmill_common::{db::UserDB, error::Error, utils::StripPath}; +use windmill_common::{ + db::UserDB, + error::{to_anyhow, Error, Result}, + utils::StripPath, +}; mod bool; mod converter; mod handler; @@ -47,76 +50,184 @@ const ERROR_REPLICATION_SLOT_NOT_EXISTS: &str = r#"The replication slot associat const ERROR_PUBLICATION_NAME_NOT_EXISTS: &str = r#"The publication associated with this trigger no longer exists. Recreate a new publication or select an existing one in the advanced tab, or delete and recreate a new trigger"#; -pub async fn get_database_connection( +fn build_tls_connector( + ssl_mode: SslMode, + root_certificate_pem: Option<&String>, +) -> Result> { + let get_tls_builder_for_verify = |root_certificate: Option<&String>| { + let mut builder = TlsConnector::builder(); + if let Some(root_certificate) = root_certificate { + let root_certificate_pem = + Certificate::from_pem(root_certificate.as_bytes()).map_err(to_anyhow)?; + builder.add_root_certificate(root_certificate_pem); + } + Ok::<_, Error>(builder) + }; + let connector = match ssl_mode { + SslMode::Disable => return Ok(None), + SslMode::Require | SslMode::Prefer => { + let mut builder = TlsConnector::builder(); + builder.danger_accept_invalid_certs(true); + builder.danger_accept_invalid_hostnames(true); + builder + } + + SslMode::VerifyCa => { + let mut builder = get_tls_builder_for_verify(root_certificate_pem)?; + builder.danger_accept_invalid_hostnames(true); + builder + } + + SslMode::VerifyFull => { + let builder = get_tls_builder_for_verify(root_certificate_pem)?; + builder + } + _ => unreachable!(), + }; + + Ok(Some(MakeTlsConnector::new( + connector.build().map_err(to_anyhow)?, + ))) +} + +pub async fn get_raw_postgres_connection( + database: &Postgres, + logical_mode: bool, +) -> Result { + let ssl_mode = match database.sslmode.as_ref() { + "disable" => SslMode::Disable, + "" | "prefer" | "allow" => SslMode::Prefer, + "require" => SslMode::Require, + "verify-ca" => SslMode::VerifyCa, + "verify-full" => SslMode::VerifyFull, + ssl_mode => { + return Err(Error::BadRequest( + format!("Invalid ssl mode for postgres: {}, please put a valid ssl_mode among the following available ssl mode: ['disable', 'allow', 'prefer', 'verify-ca', 'verify-full']", ssl_mode), + )) + } + }; + + let mut config = Config::new(); + config + .dbname(&database.dbname) + .host(&database.host) + .user(&database.user) + .ssl_mode(ssl_mode); + + if logical_mode { + config.replication_mode(rust_postgres::config::ReplicationMode::Logical); + } + + if let Some(port) = database.port { + config.port(port); + }; + + if !database.password.is_empty() { + config.password(&database.password); + } + + let connector = build_tls_connector(ssl_mode, database.root_certificate_pem.as_ref())?; + + let client = if let Some(connector) = connector { + let (client, connection) = config.connect(connector).await.map_err(to_anyhow)?; + tokio::spawn(async move { + if let Err(e) = connection.await { + tracing::debug!("{:#?}", e); + }; + tracing::info!("Successfully Connected into database"); + }); + client + } else { + let (client, connection) = config.connect(NoTls).await.map_err(to_anyhow)?; + tokio::spawn(async move { + if let Err(e) = connection.await { + tracing::debug!("{:#?}", e); + }; + tracing::info!("Successfully Connected into database"); + }); + client + }; + + Ok(client) +} + +pub async fn get_pg_connection( authed: ApiAuthed, user_db: Option, db: &DB, postgres_resource_path: &str, w_id: &str, -) -> std::result::Result { + logical_mode: bool, +) -> Result { let database = try_get_resource_from_db_as::(authed, user_db, db, postgres_resource_path, w_id) .await?; - Ok(get_raw_postgres_connection(&database).await?) + Ok(get_raw_postgres_connection(&database, logical_mode).await?) } -pub async fn get_raw_postgres_connection( - db: &Postgres, -) -> std::result::Result { - let options = { - let sslmode = if !db.sslmode.is_empty() { - PgSslMode::from_str(&db.sslmode)? - } else { - PgSslMode::Prefer - }; - let options = { - let inner_options = PgConnectOptions::new() - .host(&db.host) - .database(&db.dbname) - .ssl_mode(sslmode) - .username(&db.user); +pub async fn get_default_pg_connection( + authed: ApiAuthed, + user_db: Option, + db: &DB, + postgres_resource_path: &str, + w_id: &str, +) -> Result { + get_pg_connection(authed, user_db, db, postgres_resource_path, w_id, false).await +} - if let Some(port) = db.port { - inner_options.port(port) - } else { - inner_options - } - }; +pub async fn create_logical_replication_slot(tx: &Client, slot_name: &str) -> Result<()> { + tx.execute( + &format!("SELECT * FROM pg_create_logical_replication_slot($1, 'pgoutput')"), + &[&slot_name], + ) + .await + .map_err(to_anyhow)?; + Ok(()) +} - let options = if let Some(root_certificate_pem) = &db.root_certificate_pem { - options.ssl_root_cert_from_pem(root_certificate_pem.as_bytes().to_vec()) - } else { - options - }; +async fn check_if_valid_publication_for_postgres_version( + pg_connection: &Client, + table_to_track: Option<&[Relations]>, +) -> Result { + let postgres_version = get_postgres_version_internal(pg_connection).await?; - if !db.password.is_empty() { - options.password(&db.password) - } else { - options + let pg_14 = postgres_version.starts_with("14"); + if pg_14 { + let unsupported_publication = table_to_track + .and_then(|relations| { + relations.iter().find(|relation| { + let invalid_relation = relation.table_to_track.iter().find(|table_to_track| { + table_to_track.where_clause.is_some() + || table_to_track.columns_name.is_some() + }); + + relation.table_to_track.is_empty() || invalid_relation.is_some() + }) + }) + .is_some(); + + if unsupported_publication { + return Err(Error::BadRequest( + "Your PostgreSQL database is running version 14, which does not support the following publication features: \ + - WHERE clause filtering, \ + - selective column tracking, and \ + - tracking all tables within a schema.\n\ + These features are only available in PostgreSQL 15 and above.".to_string(), + )); } - }; - Ok(PgConnection::connect_with(&options).await?) + } + Ok(pg_14) } -pub fn create_logical_replication_slot_query(name: &str) -> String { - let query = format!( - r#" - SELECT - * - FROM - pg_create_logical_replication_slot({}, 'pgoutput');"#, - quote_literal(&name) - ); - - query -} - -pub fn create_publication_query( +pub async fn create_pg_publication( + pg_connection: &Client, publication_name: &str, table_to_track: Option<&[Relations]>, - transaction_to_track: &[&str], -) -> String { + transaction_to_track: &[String], +) -> Result<()> { + let pg_14 = + check_if_valid_publication_for_postgres_version(pg_connection, table_to_track).await?; let mut query = String::from("CREATE PUBLICATION "); query.push_str("e_identifier(publication_name)); @@ -124,21 +235,26 @@ pub fn create_publication_query( match table_to_track { Some(database_component) if !database_component.is_empty() => { query.push_str(" FOR"); + let mut first = true; for (i, schema) in database_component.iter().enumerate() { if schema.table_to_track.is_empty() { query.push_str(" TABLES IN SCHEMA "); query.push_str("e_identifier(&schema.schema_name)); } else { - query.push_str(" TABLE ONLY "); + if pg_14 && first { + query.push_str(" TABLE ONLY "); + first = false + } else if !pg_14 { + query.push_str(" TABLE ONLY "); + } for (j, table) in schema.table_to_track.iter().enumerate() { let table_name = quote_identifier(&table.table_name); let schema_name = quote_identifier(&schema.schema_name); let full_name = format!("{}.{}", &schema_name, &table_name); query.push_str(&full_name); - if !table.columns_name.is_empty() { + if let Some(columns) = table.columns_name.as_ref() { query.push_str(" ("); - let columns = table - .columns_name + let columns = columns .iter() .map(|column| quote_identifier(column)) .join(", "); @@ -174,22 +290,24 @@ pub fn create_publication_query( query.push_str("');"); } - query + pg_connection + .execute(&query, &[]) + .await + .map_err(to_anyhow)?; + Ok(()) } -pub fn drop_publication_query(publication_name: &str) -> String { +pub async fn drop_publication(pg_connection: &Client, publication_name: &str) -> Result<()> { let mut query = String::from("DROP PUBLICATION IF EXISTS "); let quoted_publication_name = quote_identifier(publication_name); query.push_str("ed_publication_name); - query.push_str(";"); - query -} -pub fn drop_logical_replication_slot_query(replication_slot_name: &str) -> String { - format!( - "SELECT pg_drop_replication_slot({});", - quote_literal(&replication_slot_name) - ) + pg_connection + .execute(&query, &[]) + .await + .map_err(to_anyhow)?; + + Ok(()) } pub fn generate_random_string() -> String { @@ -228,6 +346,10 @@ fn slot_service() -> Router { .route("/delete/*path", delete(drop_slot_name)) } +fn postgres_service() -> Router { + Router::new().route("/version/*path", get(get_postgres_version)) +} + pub fn workspaced_service() -> Router { Router::new() .route("/test", post(test_postgres_connection)) @@ -246,6 +368,7 @@ pub fn workspaced_service() -> Router { ) .nest("/publication", publication_service()) .nest("/slot", slot_service()) + .nest("/postgres", postgres_service()) } async fn run_job( diff --git a/backend/windmill-api/src/postgres_triggers/trigger.rs b/backend/windmill-api/src/postgres_triggers/trigger.rs index 71daffb4a9..4598ca1a18 100644 --- a/backend/windmill-api/src/postgres_triggers/trigger.rs +++ b/backend/windmill-api/src/postgres_triggers/trigger.rs @@ -19,25 +19,27 @@ use crate::{ use bytes::{BufMut, Bytes, BytesMut}; use chrono::TimeZone; use futures::{pin_mut, SinkExt, StreamExt}; -use native_tls::{Certificate, TlsConnector}; use pg_escape::{quote_identifier, quote_literal}; use rand::seq::SliceRandom; -use rust_postgres::{config::SslMode, Client, Config, CopyBothDuplex, NoTls, SimpleQueryMessage}; -use rust_postgres_native_tls::MakeTlsConnector; +use rust_postgres::{Client, CopyBothDuplex, SimpleQueryMessage}; use serde::Deserialize; use serde_json::value::RawValue; use sqlx::types::Json as SqlxJson; use windmill_common::{ - db::UserDB, error, triggers::TriggerKind, utils::report_critical_error, worker::to_raw_value, + db::UserDB, + error::{self, to_anyhow}, + triggers::TriggerKind, + utils::report_critical_error, + worker::to_raw_value, INSTANCE_NAME, }; use super::{ - drop_logical_replication_slot_query, drop_publication_query, get_database_connection, - handler::{Postgres, PostgresTrigger}, + drop_publication, get_default_pg_connection, get_raw_postgres_connection, + handler::{drop_logical_replication_slot, Postgres, PostgresTrigger}, replication_message::PrimaryKeepAliveBody, - ERROR_PUBLICATION_NAME_NOT_EXISTS, ERROR_REPLICATION_SLOT_NOT_EXISTS, + Error, ERROR_PUBLICATION_NAME_NOT_EXISTS, ERROR_REPLICATION_SLOT_NOT_EXISTS, }; pub struct LogicalReplicationSettings { @@ -69,109 +71,11 @@ impl RowExist for Vec { } } -#[derive(thiserror::Error, Debug)] -enum Error { - #[error("Error from database: {0}")] - Postgres(#[from] rust_postgres::Error), - #[error("Error : {0}")] - Common(#[from] windmill_common::error::Error), - #[error("Tls Error: {0}")] - Tls(#[from] native_tls::Error), -} - -fn build_tls_connector( - ssl_mode: SslMode, - root_certificate_pem: Option<&String>, -) -> Result, Error> { - let get_tls_builder_for_verify = |root_certificate: Option<&String>| { - let mut builder = TlsConnector::builder(); - if let Some(root_certificate) = root_certificate { - let root_certificate_pem = Certificate::from_pem(root_certificate.as_bytes()).map_err(|e| { - Error::Common(error::Error::BadConfig(format!("Invalid Certs: {e:#}"))) - })?; - builder.add_root_certificate(root_certificate_pem); - } - Ok::<_, Error>(builder) - }; - let connector = match ssl_mode { - SslMode::Disable => return Ok(None), - SslMode::Require | SslMode::Prefer => { - let mut builder = TlsConnector::builder(); - builder.danger_accept_invalid_certs(true); - builder.danger_accept_invalid_hostnames(true); - builder - } - - SslMode::VerifyCa => { - let mut builder = get_tls_builder_for_verify(root_certificate_pem)?; - builder.danger_accept_invalid_hostnames(true); - builder - } - - SslMode::VerifyFull => { - let builder = get_tls_builder_for_verify(root_certificate_pem)?; - builder - } - _ => unreachable!(), - }; - - Ok(Some(MakeTlsConnector::new(connector.build()?))) -} - pub struct PostgresSimpleClient(Client); impl PostgresSimpleClient { async fn new(database: &Postgres) -> Result { - let ssl_mode = match database.sslmode.as_ref() { - "disable" => SslMode::Disable, - "" | "prefer" | "allow" => SslMode::Prefer, - "require" => SslMode::Require, - "verify-ca" => SslMode::VerifyCa, - "verify-full" => SslMode::VerifyFull, - ssl_mode => { - return Err(Error::Common(windmill_common::error::Error::BadRequest( - format!("Invalid ssl mode for postgres: {}, please put a valid ssl_mode among the following avalible ssl mode: ['disable', 'allow', 'prefer', 'verify-ca', 'verify-full']", ssl_mode), - ))) - } - }; - - let mut config = Config::new(); - config - .dbname(&database.dbname) - .host(&database.host) - .user(&database.user) - .ssl_mode(ssl_mode) - .replication_mode(rust_postgres::config::ReplicationMode::Logical); - - if let Some(port) = database.port { - config.port(port); - }; - - if !database.password.is_empty() { - config.password(&database.password); - } - - let connector = build_tls_connector(ssl_mode, database.root_certificate_pem.as_ref())?; - - let client = if let Some(connector) = connector { - let (client, connection) = config.connect(connector).await?; - tokio::spawn(async move { - if let Err(e) = connection.await { - tracing::debug!("{:#?}", e); - }; - tracing::info!("Successfully Connected into database"); - }); - client - } else { - let (client, connection) = config.connect(NoTls).await?; - tokio::spawn(async move { - if let Err(e) = connection.await { - tracing::debug!("{:#?}", e); - }; - tracing::info!("Successfully Connected into database"); - }); - client - }; + let client = get_raw_postgres_connection(database, true).await?; Ok(PostgresSimpleClient(client)) } @@ -202,7 +106,8 @@ impl PostgresSimpleClient { Ok(( self.0 .copy_both_simple::(query.as_str()) - .await?, + .await + .map_err(to_anyhow)?, LogicalReplicationSettings::new(false), )) } @@ -527,12 +432,13 @@ impl PostgresConfig { "SELECT pubname FROM pg_publication WHERE pubname = {}", quote_literal(&publication_name) )) - .await?; + .await + .map_err(to_anyhow)?; if !publication.row_exist() { - return Err(Error::Common(error::Error::BadConfig( + return Err(Error::BadConfig( ERROR_PUBLICATION_NAME_NOT_EXISTS.to_string(), - ))); + )); } let replication_slot = client @@ -540,17 +446,19 @@ impl PostgresConfig { "SELECT slot_name FROM pg_replication_slots WHERE slot_name = {}", quote_literal(&replication_slot_name) )) - .await?; + .await + .map_err(to_anyhow)?; if !replication_slot.row_exist() { - return Err(Error::Common(error::Error::BadConfig( + return Err(Error::BadConfig( ERROR_REPLICATION_SLOT_NOT_EXISTS.to_string(), - ))); + )); } let (logical_replication_stream, logical_replication_settings) = client .get_logical_replication_stream(&publication_name, &replication_slot_name) - .await?; + .await + .map_err(to_anyhow)?; Ok((logical_replication_stream, logical_replication_settings)) } @@ -585,7 +493,7 @@ impl PostgresConfig { let user_db = UserDB::new(db.clone()); - let mut connection = get_database_connection( + let mut pg_connection = get_default_pg_connection( authed.clone(), Some(user_db.clone()), &db, @@ -594,13 +502,12 @@ impl PostgresConfig { ) .await?; - let query = drop_logical_replication_slot_query(replication_slot_name); + if capture.trigger_config.basic_mode.unwrap_or(false) { + drop_logical_replication_slot(&mut pg_connection, replication_slot_name) + .await?; - let _ = sqlx::query(&query).execute(&mut connection).await; - - let query = drop_publication_query(publication_name); - - let _ = sqlx::query(&query).execute(&mut connection).await; + drop_publication(&mut pg_connection, publication_name).await?; + } Ok(()) } diff --git a/backend/windmill-api/src/raw_apps.rs b/backend/windmill-api/src/raw_apps.rs index bb2763b73d..3a895bab97 100644 --- a/backend/windmill-api/src/raw_apps.rs +++ b/backend/windmill-api/src/raw_apps.rs @@ -22,7 +22,7 @@ use serde::{Deserialize, Serialize}; use sql_builder::{bind::Bind, SqlBuilder}; use sqlx::FromRow; use std::str; -use windmill_audit::audit_ee::audit_log; +use windmill_audit::audit_oss::audit_log; use windmill_audit::ActionKind; use windmill_common::{ apps::ListAppQuery, diff --git a/backend/windmill-api/src/resources.rs b/backend/windmill-api/src/resources.rs index 8a2fab0434..d2d58c918d 100644 --- a/backend/windmill-api/src/resources.rs +++ b/backend/windmill-api/src/resources.rs @@ -26,7 +26,7 @@ use serde_json::{value::RawValue, Value}; use sql_builder::{bind::Bind, quote, SqlBuilder}; use sqlx::{FromRow, Postgres, Transaction}; use uuid::Uuid; -use windmill_audit::audit_ee::{audit_log, AuditAuthor}; +use windmill_audit::audit_oss::{audit_log, AuditAuthor}; use windmill_audit::ActionKind; use windmill_common::{ db::UserDB, diff --git a/backend/windmill-api/src/saml_ee.rs b/backend/windmill-api/src/saml_oss.rs similarity index 75% rename from backend/windmill-api/src/saml_ee.rs rename to backend/windmill-api/src/saml_oss.rs index b3f1d4653c..0f4f4aa7f6 100644 --- a/backend/windmill-api/src/saml_ee.rs +++ b/backend/windmill-api/src/saml_oss.rs @@ -7,18 +7,27 @@ */ #![allow(non_snake_case)] +#[cfg(feature = "private")] +#[allow(unused)] +pub use crate::saml_ee::*; + +#[cfg(not(feature = "private"))] use axum::{routing::post, Router}; +#[cfg(not(feature = "private"))] pub struct ServiceProviderExt(); +#[cfg(not(feature = "private"))] pub async fn build_sp_extension() -> anyhow::Result { return Ok(ServiceProviderExt()); } +#[cfg(not(feature = "private"))] pub fn global_service() -> Router { Router::new().route("/acs", post(acs)) } +#[cfg(not(feature = "private"))] pub async fn acs() -> String { // Implementation is not open source as it is a Windmill Enterprise Edition feature "SAML available only in enterprise version".to_string() diff --git a/backend/windmill-api/src/schedule.rs b/backend/windmill-api/src/schedule.rs index 90d91c846c..3cb13798d9 100644 --- a/backend/windmill-api/src/schedule.rs +++ b/backend/windmill-api/src/schedule.rs @@ -22,7 +22,7 @@ use serde::{Deserialize, Serialize}; use sql_builder::{prelude::Bind, SqlBuilder}; use sqlx::{Postgres, Transaction}; use std::str::FromStr; -use windmill_audit::audit_ee::audit_log; +use windmill_audit::audit_oss::audit_log; use windmill_audit::ActionKind; use windmill_common::{ db::UserDB, diff --git a/backend/windmill-api/src/scim_ee.rs b/backend/windmill-api/src/scim_oss.rs similarity index 72% rename from backend/windmill-api/src/scim_ee.rs rename to backend/windmill-api/src/scim_oss.rs index f11097f874..5210411466 100644 --- a/backend/windmill-api/src/scim_ee.rs +++ b/backend/windmill-api/src/scim_oss.rs @@ -1,3 +1,7 @@ +#[cfg(feature = "private")] +#[allow(unused)] +pub use crate::scim_ee::*; + /* * Author: Ruben Fiszel * Copyright: Windmill Labs, Inc 2023 @@ -6,17 +10,22 @@ * LICENSE-AGPL for a copy of the license. */ +#[cfg(not(feature = "private"))] use axum::{middleware::Next, response::Response, routing::get, Router}; +#[cfg(not(feature = "private"))] use hyper::Request; +#[cfg(not(feature = "private"))] pub fn global_service() -> Router { Router::new().route("/ee", get(ee)) } +#[cfg(not(feature = "private"))] pub async fn ee() -> String { return "Enterprise Edition".to_string(); } +#[cfg(not(feature = "private"))] pub async fn has_scim_token(_request: Request, _next: Next) -> Response { //Not implemented in open-source version todo!() diff --git a/backend/windmill-api/src/scripts.rs b/backend/windmill-api/src/scripts.rs index a827fa5079..8ac824e4c1 100644 --- a/backend/windmill-api/src/scripts.rs +++ b/backend/windmill-api/src/scripts.rs @@ -38,7 +38,7 @@ use std::{ hash::{Hash, Hasher}, sync::Arc, }; -use windmill_audit::audit_ee::audit_log; +use windmill_audit::audit_oss::audit_log; use windmill_audit::ActionKind; use windmill_worker::process_relative_imports; @@ -206,7 +206,6 @@ async fn list_scripts( Query(lq): Query, ) -> JsonResult> { let (per_page, offset) = paginate(pagination); - let mut sqlb = SqlBuilder::select_from("script as o") .fields(&[ "hash", @@ -321,6 +320,16 @@ async fn list_scripts( .fields(&["dm.deployment_msg"]); } + if let Some(languages) = lq.languages { + sqlb.and_where_in( + "language", + &languages + .iter() + .map(|language| quote(language.as_str())) + .collect_vec(), + ); + } + let sql = sqlb.sql().map_err(|e| Error::internal_err(e.to_string()))?; let mut tx = user_db.begin(&authed).await?; let rows = sqlx::query_as::<_, ListableScript>(&sql) @@ -1280,6 +1289,11 @@ async fn raw_script_by_path_unpinned( raw_script_by_path_internal(path, user_db, db, authed, w_id, true).await } +lazy_static::lazy_static! { + static ref DEBUG_RAW_SCRIPT_ENDPOINTS: bool = + std::env::var("DEBUG_RAW_SCRIPT_ENDPOINTS").is_ok(); +} + async fn raw_script_by_path_internal( path: StripPath, user_db: UserDB, @@ -1332,6 +1346,19 @@ async fn raw_script_by_path_internal( "Script {path} exists but {} does not have permissions to access it", authed.username ))); + } else { + if *DEBUG_RAW_SCRIPT_ENDPOINTS { + let other_script_o = sqlx::query_scalar!( + "SELECT path FROM script WHERE workspace_id = $1 AND archived = false", + w_id + ) + .fetch_all(&db) + .await?; + tracing::info!( + "Script {path} does not exist in workspace {w_id} but these paths do: {:?}", + other_script_o.join(", ") + ) + } } } diff --git a/backend/windmill-api/src/settings.rs b/backend/windmill-api/src/settings.rs index b589713b91..9dfb687e51 100644 --- a/backend/windmill-api/src/settings.rs +++ b/backend/windmill-api/src/settings.rs @@ -10,7 +10,7 @@ use std::time::Duration; use crate::{ db::{ApiAuthed, DB}, - ee::validate_license_key, + ee_oss::validate_license_key, utils::{generate_instance_username_for_all_users, require_super_admin}, HTTP_CLIENT, }; @@ -29,9 +29,9 @@ use crate::utils::require_devops_role; use serde::Deserialize; #[cfg(feature = "enterprise")] -use windmill_common::ee::{send_critical_alert, CriticalAlertKind, CriticalErrorChannel}; +use windmill_common::ee_oss::{send_critical_alert, CriticalAlertKind, CriticalErrorChannel}; use windmill_common::{ - email_ee::send_email, + email_oss::send_email, error::{self, JsonResult, Result}, global_settings::{ AUTOMATE_USERNAME_CREATION_SETTING, CRITICAL_ALERT_MUTE_UI_SETTING, EMAIL_DOMAIN_SETTING, @@ -326,10 +326,10 @@ async fn list_global_settings() -> JsonResult { pub async fn send_stats(Extension(db): Extension, authed: ApiAuthed) -> Result { require_super_admin(&db, &authed.email).await?; - windmill_common::stats_ee::send_stats( + windmill_common::stats_oss::send_stats( &HTTP_CLIENT, &db, - windmill_common::stats_ee::SendStatsReason::Manual, + windmill_common::stats_oss::SendStatsReason::Manual, ) .await?; @@ -390,11 +390,11 @@ pub async fn renew_license_key( authed: ApiAuthed, ) -> Result { require_super_admin(&db, &authed.email).await?; - let result = windmill_common::ee::renew_license_key( + let result = windmill_common::ee_oss::renew_license_key( &HTTP_CLIENT, &db, license_key, - windmill_common::ee::RenewReason::Manual, + windmill_common::ee_oss::RenewReason::Manual, ) .await; @@ -424,7 +424,7 @@ pub async fn create_customer_portal_session( Query(LicenseQuery { license_key }): Query, ) -> Result { let url = - windmill_common::ee::create_customer_portal_session(&HTTP_CLIENT, license_key).await?; + windmill_common::ee_oss::create_customer_portal_session(&HTTP_CLIENT, license_key).await?; return Ok(url); } diff --git a/backend/windmill-api/src/smtp_server_ee.rs b/backend/windmill-api/src/smtp_server_oss.rs similarity index 69% rename from backend/windmill-api/src/smtp_server_ee.rs rename to backend/windmill-api/src/smtp_server_oss.rs index 48e274f6a1..7d2b72c7f1 100644 --- a/backend/windmill-api/src/smtp_server_ee.rs +++ b/backend/windmill-api/src/smtp_server_oss.rs @@ -1,7 +1,15 @@ +#[cfg(feature = "private")] +#[allow(unused)] +pub use crate::smtp_server_ee::*; + +#[cfg(not(feature = "private"))] use crate::{auth::AuthCache, db::DB}; +#[cfg(not(feature = "private"))] use std::{net::SocketAddr, sync::Arc}; +#[cfg(not(feature = "private"))] use windmill_common::db::UserDB; +#[cfg(not(feature = "private"))] pub struct SmtpServer { pub auth_cache: Arc, pub db: DB, @@ -9,6 +17,7 @@ pub struct SmtpServer { pub base_internal_url: String, } +#[cfg(not(feature = "private"))] impl SmtpServer { pub async fn start_listener_thread(self: Arc, _addr: SocketAddr) -> anyhow::Result<()> { let _ = self.auth_cache; diff --git a/backend/windmill-api/src/sqs_triggers_ee.rs b/backend/windmill-api/src/sqs_triggers_oss.rs similarity index 75% rename from backend/windmill-api/src/sqs_triggers_ee.rs rename to backend/windmill-api/src/sqs_triggers_oss.rs index da9b288f6b..6e01641a5e 100644 --- a/backend/windmill-api/src/sqs_triggers_ee.rs +++ b/backend/windmill-api/src/sqs_triggers_oss.rs @@ -1,18 +1,28 @@ +#[cfg(feature = "private")] +#[allow(unused)] +pub use crate::sqs_triggers_ee::*; + +#[cfg(not(feature = "private"))] use crate::db::DB; +#[cfg(not(feature = "private"))] use axum::Router; +#[cfg(not(feature = "private"))] use serde::{Deserialize, Serialize}; +#[cfg(not(feature = "private"))] use windmill_common::auth::aws::AwsAuthResourceType; - +#[cfg(not(feature = "private"))] pub fn workspaced_service() -> Router { Router::new() } +#[cfg(not(feature = "private"))] pub fn start_sqs(_db: DB, mut _killpill_rx: tokio::sync::broadcast::Receiver<()>) -> () { // implementation is not open source } #[derive(Debug, Clone, Deserialize, Serialize)] +#[cfg(not(feature = "private"))] pub struct SqsTrigger { pub queue_url: String, pub aws_auth_resource_type: AwsAuthResourceType, @@ -30,4 +40,4 @@ pub struct SqsTrigger { pub server_id: Option, pub last_server_ping: Option>, pub enabled: bool, -} \ No newline at end of file +} diff --git a/backend/windmill-api/src/stripe_ee.rs b/backend/windmill-api/src/stripe_ee.rs deleted file mode 100644 index 6934cad3bc..0000000000 --- a/backend/windmill-api/src/stripe_ee.rs +++ /dev/null @@ -1,5 +0,0 @@ -use axum::Router; - -pub fn add_stripe_routes(router: Router) -> Router { - return router; -} diff --git a/backend/windmill-api/src/stripe_oss.rs b/backend/windmill-api/src/stripe_oss.rs new file mode 100644 index 0000000000..206a78dc53 --- /dev/null +++ b/backend/windmill-api/src/stripe_oss.rs @@ -0,0 +1,11 @@ +#[cfg(feature = "private")] +#[allow(unused)] +pub use crate::stripe_ee::*; + +#[cfg(not(feature = "private"))] +use axum::Router; + +#[cfg(not(feature = "private"))] +pub fn add_stripe_routes(router: Router) -> Router { + return router; +} diff --git a/backend/windmill-api/src/teams_approvals_ee.rs b/backend/windmill-api/src/teams_approvals_oss.rs similarity index 51% rename from backend/windmill-api/src/teams_approvals_ee.rs rename to backend/windmill-api/src/teams_approvals_oss.rs index 05698b98e3..df5826aa71 100644 --- a/backend/windmill-api/src/teams_approvals_ee.rs +++ b/backend/windmill-api/src/teams_approvals_oss.rs @@ -1,7 +1,14 @@ +#[cfg(feature = "private")] +#[allow(unused)] +pub use crate::teams_approvals_ee::*; + +#[cfg(not(feature = "private"))] use hyper::StatusCode; +#[cfg(not(feature = "private"))] use windmill_common::error::Error; +#[cfg(not(feature = "private"))] pub async fn request_teams_approval() -> Result { Err(Error::InternalErr("enterprise feature only".to_string())) -} \ No newline at end of file +} diff --git a/backend/windmill-api/src/teams_ee.rs b/backend/windmill-api/src/teams_oss.rs similarity index 70% rename from backend/windmill-api/src/teams_ee.rs rename to backend/windmill-api/src/teams_oss.rs index 46cbe72059..95d4883690 100644 --- a/backend/windmill-api/src/teams_ee.rs +++ b/backend/windmill-api/src/teams_oss.rs @@ -1,39 +1,50 @@ -use http::status::StatusCode; -#[cfg(feature = "enterprise")] +#[cfg(feature = "private")] +#[allow(unused)] +pub use crate::teams_ee::*; + +#[cfg(all(feature = "enterprise", not(feature = "private")))] use axum::Router; +#[cfg(not(feature = "private"))] +use http::status::StatusCode; +#[cfg(not(feature = "private"))] use windmill_common::error::Error; +#[cfg(not(feature = "private"))] pub async fn edit_teams_command() -> Result { return Err(Error::BadRequest( "Teams only available on enterprise".to_string(), )); } +#[cfg(not(feature = "private"))] pub async fn workspaces_list_available_teams_ids() -> Result { return Err(Error::BadRequest( "Teams only available on enterprise".to_string(), )); } +#[cfg(not(feature = "private"))] pub async fn connect_teams() -> Result { return Err(Error::BadRequest( "Teams only available on enterprise".to_string(), )); } +#[cfg(not(feature = "private"))] pub async fn run_teams_message_test_job() -> Result { return Err(Error::BadRequest( "Teams only available on enterprise".to_string(), )); } +#[cfg(not(feature = "private"))] pub async fn workspaces_list_available_teams_channels() -> Result { return Err(Error::BadRequest( "Teams only available on enterprise".to_string(), )); } -#[cfg(feature = "enterprise")] +#[cfg(all(feature = "enterprise", not(feature = "private")))] pub fn teams_service() -> Router { Router::new() -} \ No newline at end of file +} diff --git a/backend/windmill-api/src/triggers.rs b/backend/windmill-api/src/triggers.rs index 7101bdd2a9..c1cccb1c98 100644 --- a/backend/windmill-api/src/triggers.rs +++ b/backend/windmill-api/src/triggers.rs @@ -21,6 +21,9 @@ pub struct TriggersCount { kafka_count: i64, nats_count: i64, postgres_count: i64, + mqtt_count: i64, + sqs_count: i64, + gcp_count: i64, } pub(crate) async fn get_triggers_count_internal( db: &DB, @@ -97,6 +100,36 @@ pub(crate) async fn get_triggers_count_internal( .await? .unwrap_or(0); + let mqtt_count = sqlx::query_scalar!( + "SELECT COUNT(*) FROM mqtt_trigger WHERE script_path = $1 AND is_flow = $2 AND workspace_id = $3", + path, + is_flow, + w_id + ) + .fetch_one(db) + .await? + .unwrap_or(0); + + let sqs_count = sqlx::query_scalar!( + "SELECT COUNT(*) FROM sqs_trigger WHERE script_path = $1 AND is_flow = $2 AND workspace_id = $3", + path, + is_flow, + w_id + ) + .fetch_one(db) + .await? + .unwrap_or(0); + + let gcp_count = sqlx::query_scalar!( + "SELECT COUNT(*) FROM gcp_trigger WHERE script_path = $1 AND is_flow = $2 AND workspace_id = $3", + path, + is_flow, + w_id + ) + .fetch_one(db) + .await? + .unwrap_or(0); + let webhook_count = (if is_flow { sqlx::query_scalar!( "SELECT COUNT(*) FROM token WHERE label LIKE 'webhook-%' AND workspace_id = $1 AND scopes @> ARRAY['run:flow/' || $2]::text[]", @@ -141,6 +174,9 @@ pub(crate) async fn get_triggers_count_internal( kafka_count, nats_count, postgres_count, + mqtt_count, + gcp_count, + sqs_count, })) } diff --git a/backend/windmill-api/src/users.rs b/backend/windmill-api/src/users.rs index 1f8529fe6a..da77337703 100644 --- a/backend/windmill-api/src/users.rs +++ b/backend/windmill-api/src/users.rs @@ -42,7 +42,7 @@ use sqlx::FromRow; use time::OffsetDateTime; use tower_cookies::{Cookie, Cookies}; use tracing::Instrument; -use windmill_audit::audit_ee::{audit_log, AuditAuthor}; +use windmill_audit::audit_oss::{audit_log, AuditAuthor}; use windmill_audit::ActionKind; use windmill_common::auth::fetch_authed_from_permissioned_as; use windmill_common::global_settings::AUTOMATE_USERNAME_CREATION_SETTING; @@ -334,6 +334,7 @@ pub struct NewUser { pub super_admin: bool, pub name: Option, pub company: Option, + pub skip_email: Option, } #[derive(Deserialize)] @@ -1507,7 +1508,7 @@ async fn create_user( Extension(argon2): Extension>>, Json(nu): Json, ) -> Result<(StatusCode, String)> { - crate::users_ee::create_user(authed, db, webhook, argon2, nu).await + crate::users_oss::create_user(authed, db, webhook, argon2, nu).await } async fn delete_workspace_user( @@ -1581,7 +1582,7 @@ async fn set_password( Json(ep): Json, ) -> Result { let email = authed.email.clone(); - crate::users_ee::set_password(db, argon2, authed, &email, ep).await + crate::users_oss::set_password(db, argon2, authed, &email, ep).await } async fn set_password_of_user( @@ -1592,7 +1593,7 @@ async fn set_password_of_user( Json(ep): Json, ) -> Result { require_super_admin(&db, &authed.email).await?; - crate::users_ee::set_password(db, argon2, authed, &email, ep).await + crate::users_oss::set_password(db, argon2, authed, &email, ep).await } async fn set_login_type( diff --git a/backend/windmill-api/src/users_ee.rs b/backend/windmill-api/src/users_oss.rs similarity index 71% rename from backend/windmill-api/src/users_ee.rs rename to backend/windmill-api/src/users_oss.rs index 7a11239a2f..88a12eb710 100644 --- a/backend/windmill-api/src/users_ee.rs +++ b/backend/windmill-api/src/users_oss.rs @@ -1,15 +1,27 @@ +#[cfg(feature = "private")] +#[allow(unused)] +pub use crate::users_ee::*; + +#[cfg(not(feature = "private"))] use std::sync::Arc; +#[cfg(not(feature = "private"))] use crate::db::ApiAuthed; +#[cfg(not(feature = "private"))] use crate::users::{EditPassword, NewUser}; +#[cfg(not(feature = "private"))] use crate::{db::DB, webhook_util::WebhookShared}; +#[cfg(not(feature = "private"))] use argon2::Argon2; +#[cfg(not(feature = "private"))] use http::StatusCode; +#[cfg(not(feature = "private"))] use windmill_common::error::{Error, Result}; +#[cfg(not(feature = "private"))] pub async fn create_user( _authed: ApiAuthed, _db: DB, @@ -22,6 +34,7 @@ pub async fn create_user( )) } +#[cfg(not(feature = "private"))] pub async fn set_password( _db: DB, _argon2: Arc>, @@ -34,6 +47,7 @@ pub async fn set_password( )) } +#[cfg(not(feature = "private"))] pub fn send_email_if_possible(_subject: &str, _content: &str, _to: &str) { tracing::warn!( "send_email_if_possible is not implemented in Windmill's Open Source repository" diff --git a/backend/windmill-api/src/variables.rs b/backend/windmill-api/src/variables.rs index e810f95750..7dc6a85b49 100644 --- a/backend/windmill-api/src/variables.rs +++ b/backend/windmill-api/src/variables.rs @@ -20,7 +20,7 @@ use axum::{ use hyper::StatusCode; use serde_json::Value; -use windmill_audit::audit_ee::{audit_log, AuditAuthorable}; +use windmill_audit::audit_oss::{audit_log, AuditAuthorable}; use windmill_audit::ActionKind; use windmill_common::{ db::UserDB, @@ -186,7 +186,7 @@ async fn get_variable( #[cfg(feature = "oauth2")] { Some( - crate::oauth2_ee::_refresh_token( + crate::oauth2_oss::_refresh_token( tx, &variable.path, &w_id, @@ -653,7 +653,7 @@ pub async fn get_value_internal<'c>( if variable.is_expired.unwrap_or(false) && variable.account.is_some() { #[cfg(feature = "oauth2")] { - crate::oauth2_ee::_refresh_token( + crate::oauth2_oss::_refresh_token( tx, &variable.path, &w_id, diff --git a/backend/windmill-api/src/websocket_triggers.rs b/backend/windmill-api/src/websocket_triggers.rs index 8bd165844c..8c263b1f62 100644 --- a/backend/windmill-api/src/websocket_triggers.rs +++ b/backend/windmill-api/src/websocket_triggers.rs @@ -20,7 +20,7 @@ use std::{collections::HashMap, fmt}; use tokio::net::TcpStream; use tokio_tungstenite::{connect_async, tungstenite::Message, MaybeTlsStream, WebSocketStream}; use uuid::Uuid; -use windmill_audit::{audit_ee::audit_log, ActionKind}; +use windmill_audit::{audit_oss::audit_log, ActionKind}; use windmill_common::{ db::UserDB, error::{self, to_anyhow, JsonResult}, diff --git a/backend/windmill-api/src/workspaces.rs b/backend/windmill-api/src/workspaces.rs index 75c9a8d034..b8e8a813f3 100644 --- a/backend/windmill-api/src/workspaces.rs +++ b/backend/windmill-api/src/workspaces.rs @@ -10,7 +10,7 @@ use std::collections::HashMap; use crate::ai::{AIConfig, AI_REQUEST_CACHE}; use crate::db::ApiAuthed; -use crate::users_ee::send_email_if_possible; +use crate::users_oss::send_email_if_possible; use crate::utils::get_instance_username_or_create_pending; use crate::BASE_URL; use crate::{ @@ -30,7 +30,7 @@ use chrono::Utc; use regex::Regex; use uuid::Uuid; -use windmill_audit::audit_ee::audit_log; +use windmill_audit::audit_oss::audit_log; use windmill_audit::ActionKind; use windmill_common::db::UserDB; use windmill_common::s3_helpers::LargeFileStorage; @@ -58,7 +58,7 @@ use sqlx::{FromRow, Postgres, Transaction}; use windmill_common::oauth2::InstanceEvent; use windmill_common::utils::not_found_if_none; -use crate::teams_ee::{ +use crate::teams_oss::{ connect_teams, edit_teams_command, run_teams_message_test_job, workspaces_list_available_teams_channels, workspaces_list_available_teams_ids, }; @@ -145,7 +145,7 @@ pub fn workspaced_service() -> Router { #[cfg(all(feature = "stripe", feature = "enterprise"))] { - crate::stripe_ee::add_stripe_routes(router) + crate::stripe_oss::add_stripe_routes(router) } #[cfg(not(feature = "stripe"))] @@ -640,7 +640,7 @@ async fn edit_auto_invite( Path(w_id): Path, Json(ea): Json, ) -> Result { - crate::workspaces_ee::edit_auto_invite(authed, db, w_id, ea).await + crate::workspaces_oss::edit_auto_invite(authed, db, w_id, ea).await } async fn edit_webhook( diff --git a/backend/windmill-api/src/workspaces_export.rs b/backend/windmill-api/src/workspaces_export.rs index cd2be0f708..b3ad6dbdd7 100644 --- a/backend/windmill-api/src/workspaces_export.rs +++ b/backend/windmill-api/src/workspaces_export.rs @@ -622,7 +622,7 @@ pub(crate) async fn tarball_workspace( #[cfg(all(feature = "enterprise", feature = "kafka"))] { let kafka_triggers = sqlx::query_as!( - crate::kafka_triggers_ee::KafkaTrigger, + crate::kafka_triggers_oss::KafkaTrigger, "SELECT * FROM kafka_trigger WHERE workspace_id = $1", &w_id @@ -644,7 +644,7 @@ pub(crate) async fn tarball_workspace( #[cfg(all(feature = "enterprise", feature = "sqs_trigger"))] { let sqs_triggers = sqlx::query_as!( - crate::sqs_triggers_ee::SqsTrigger, + crate::sqs_triggers_oss::SqsTrigger, r#" SELECT aws_auth_resource_type AS "aws_auth_resource_type: _", @@ -684,7 +684,7 @@ pub(crate) async fn tarball_workspace( #[cfg(all(feature = "enterprise", feature = "gcp_trigger"))] { let gcp_triggers = sqlx::query_as!( - crate::gcp_triggers_ee::GcpTrigger, + crate::gcp_triggers_oss::GcpTrigger, r#" SELECT gcp_resource_path, @@ -726,7 +726,7 @@ pub(crate) async fn tarball_workspace( #[cfg(all(feature = "enterprise", feature = "nats"))] { let nats_triggers = sqlx::query_as!( - crate::nats_triggers_ee::NatsTrigger, + crate::nats_triggers_oss::NatsTrigger, "SELECT * FROM nats_trigger WHERE workspace_id = $1", &w_id diff --git a/backend/windmill-api/src/workspaces_extra.rs b/backend/windmill-api/src/workspaces_extra.rs index 07570df5c3..51a224d0e2 100644 --- a/backend/windmill-api/src/workspaces_extra.rs +++ b/backend/windmill-api/src/workspaces_extra.rs @@ -8,7 +8,7 @@ use axum::{ Json, }; -use windmill_audit::audit_ee::audit_log; +use windmill_audit::audit_oss::audit_log; use windmill_audit::ActionKind; use windmill_common::worker::CLOUD_HOSTED; diff --git a/backend/windmill-api/src/workspaces_ee.rs b/backend/windmill-api/src/workspaces_oss.rs similarity index 70% rename from backend/windmill-api/src/workspaces_ee.rs rename to backend/windmill-api/src/workspaces_oss.rs index aa8799e233..4f55539da1 100644 --- a/backend/windmill-api/src/workspaces_ee.rs +++ b/backend/windmill-api/src/workspaces_oss.rs @@ -1,8 +1,14 @@ +#[cfg(feature = "private")] +#[allow(unused)] +pub use crate::workspaces_ee::*; + +#[cfg(not(feature = "private"))] use crate::{ db::{ApiAuthed, DB}, workspaces::EditAutoInvite, }; +#[cfg(not(feature = "private"))] pub async fn edit_auto_invite( _authed: ApiAuthed, _db: DB, diff --git a/backend/windmill-audit/Cargo.toml b/backend/windmill-audit/Cargo.toml index 8b202b4abf..5a9ea376b6 100644 --- a/backend/windmill-audit/Cargo.toml +++ b/backend/windmill-audit/Cargo.toml @@ -10,6 +10,7 @@ path = "./src/lib.rs" [features] enterprise = ["windmill-common/enterprise"] +private = [] [dependencies] serde.workspace = true diff --git a/backend/windmill-audit/src/audit_ee.rs b/backend/windmill-audit/src/audit_oss.rs similarity index 78% rename from backend/windmill-audit/src/audit_ee.rs rename to backend/windmill-audit/src/audit_oss.rs index 97a8e6bbcf..d29daaa7ae 100644 --- a/backend/windmill-audit/src/audit_ee.rs +++ b/backend/windmill-audit/src/audit_oss.rs @@ -1,3 +1,7 @@ +#[cfg(feature = "private")] +#[allow(unused)] +pub use crate::audit_ee::*; + /* * Author: Ruben Fiszel * Copyright: Windmill Labs, Inc 2022 @@ -5,23 +9,26 @@ * Please see the included NOTICE for copyright information and * LICENSE-AGPL for a copy of the license. */ -use std::collections::HashMap; - -use windmill_common::{ - error::{Error, Result}, - utils::Pagination, +#[cfg(not(feature = "private"))] +use { + crate::{ActionKind, AuditLog, ListAuditLogQuery}, + sqlx::{Postgres, Transaction}, + std::collections::HashMap, + windmill_common::{ + error::{Error, Result}, + utils::Pagination, + }, }; -use crate::{ActionKind, AuditLog, ListAuditLogQuery}; -use sqlx::{Postgres, Transaction}; - #[derive(Clone)] +#[cfg(not(feature = "private"))] pub struct AuditAuthor { pub username: String, pub email: String, pub username_override: Option, } +#[cfg(not(feature = "private"))] impl AuditAuthorable for AuditAuthor { fn email(&self) -> &str { &self.email @@ -36,12 +43,14 @@ impl AuditAuthorable for AuditAuthor { } } +#[cfg(not(feature = "private"))] pub trait AuditAuthorable { fn username(&self) -> &str; fn email(&self) -> &str; fn username_override(&self) -> Option<&str>; } +#[cfg(not(feature = "private"))] #[tracing::instrument(level = "trace", skip_all)] pub async fn audit_log<'c, E: sqlx::Executor<'c, Database = Postgres>>( _db: E, @@ -56,6 +65,7 @@ pub async fn audit_log<'c, E: sqlx::Executor<'c, Database = Postgres>>( Ok(()) } +#[cfg(not(feature = "private"))] pub async fn list_audit( _tx: Transaction<'_, Postgres>, _w_id: String, @@ -66,6 +76,7 @@ pub async fn list_audit( return Ok(vec![]); } +#[cfg(not(feature = "private"))] pub async fn get_audit(tx: Transaction<'_, Postgres>, _id: i32, _w_id: &str) -> Result { // Implementation is not open source as Audit logs is a Windmill Enterprise Edition feature tx.commit().await?; diff --git a/backend/windmill-audit/src/lib.rs b/backend/windmill-audit/src/lib.rs index 10894798fb..15bda522c1 100644 --- a/backend/windmill-audit/src/lib.rs +++ b/backend/windmill-audit/src/lib.rs @@ -1,7 +1,9 @@ use serde::{Deserialize, Serialize}; use sqlx::FromRow; +#[cfg(feature = "private")] pub mod audit_ee; +pub mod audit_oss; #[derive(sqlx::Type, Serialize, Deserialize, Debug)] #[sqlx(type_name = "ACTION_KIND", rename_all = "lowercase")] diff --git a/backend/windmill-autoscaling/Cargo.toml b/backend/windmill-autoscaling/Cargo.toml index fbebaf0fd7..7e4bc623ad 100644 --- a/backend/windmill-autoscaling/Cargo.toml +++ b/backend/windmill-autoscaling/Cargo.toml @@ -10,6 +10,7 @@ path = "./src/lib.rs" [features] enterprise = ["windmill-queue/enterprise", "windmill-common/enterprise"] +private = [] default = [] [dependencies] diff --git a/backend/windmill-autoscaling/src/autoscaling_ee.rs b/backend/windmill-autoscaling/src/autoscaling_ee.rs deleted file mode 100644 index 1c9defbede..0000000000 --- a/backend/windmill-autoscaling/src/autoscaling_ee.rs +++ /dev/null @@ -1,6 +0,0 @@ -use windmill_common::DB; - -pub async fn apply_all_autoscaling(_db: &DB) -> anyhow::Result<()> { - // Autoscaling is an ee feature - Ok(()) -} diff --git a/backend/windmill-autoscaling/src/autoscaling_oss.rs b/backend/windmill-autoscaling/src/autoscaling_oss.rs new file mode 100644 index 0000000000..c9cbdc3ee0 --- /dev/null +++ b/backend/windmill-autoscaling/src/autoscaling_oss.rs @@ -0,0 +1,12 @@ +#[cfg(feature = "private")] +#[allow(unused)] +pub use crate::autoscaling_ee::*; + +#[cfg(not(feature = "private"))] +use windmill_common::DB; + +#[cfg(not(feature = "private"))] +pub async fn apply_all_autoscaling(_db: &DB) -> anyhow::Result<()> { + // Autoscaling is an ee feature + Ok(()) +} diff --git a/backend/windmill-autoscaling/src/lib.rs b/backend/windmill-autoscaling/src/lib.rs index 28b9319244..1b9d6b94bf 100644 --- a/backend/windmill-autoscaling/src/lib.rs +++ b/backend/windmill-autoscaling/src/lib.rs @@ -1,2 +1,4 @@ -mod autoscaling_ee; -pub use autoscaling_ee::*; +#[cfg(feature = "private")] +pub mod autoscaling_ee; +mod autoscaling_oss; +pub use autoscaling_oss::*; diff --git a/backend/windmill-common/Cargo.toml b/backend/windmill-common/Cargo.toml index 17764b1253..6a7a3b124b 100644 --- a/backend/windmill-common/Cargo.toml +++ b/backend/windmill-common/Cargo.toml @@ -7,6 +7,7 @@ edition.workspace = true [features] default = [] enterprise = [] +private = [] jemalloc = ["dep:tikv-jemalloc-ctl"] tantivy = [] prometheus = ["dep:prometheus"] diff --git a/backend/windmill-common/src/agent_workers.rs b/backend/windmill-common/src/agent_workers.rs index 7c91ec2165..52c7e2d70c 100644 --- a/backend/windmill-common/src/agent_workers.rs +++ b/backend/windmill-common/src/agent_workers.rs @@ -36,6 +36,7 @@ pub struct AgentAuth { } pub const AGENT_JWT_PREFIX: &str = "jwt_agent_"; + pub fn build_agent_http_client(worker_suffix: &str) -> HttpClient { let client = ClientBuilder::new( reqwest::Client::builder() diff --git a/backend/windmill-common/src/client.rs b/backend/windmill-common/src/client.rs index 0c5394c6e6..6bc1d12ef9 100644 --- a/backend/windmill-common/src/client.rs +++ b/backend/windmill-common/src/client.rs @@ -17,6 +17,15 @@ pub struct AuthedClient { } impl AuthedClient { + pub fn new( + base_internal_url: String, + workspace: String, + token: String, + force_client: Option, + ) -> AuthedClient { + AuthedClient { base_internal_url, workspace, token, force_client } + } + pub async fn get(&self, url: &str, query: Vec<(&str, String)>) -> anyhow::Result { self.force_client .as_ref() diff --git a/backend/windmill-common/src/ee.rs b/backend/windmill-common/src/ee_oss.rs similarity index 63% rename from backend/windmill-common/src/ee.rs rename to backend/windmill-common/src/ee_oss.rs index 7acb430f1b..51b1efd2e2 100644 --- a/backend/windmill-common/src/ee.rs +++ b/backend/windmill-common/src/ee_oss.rs @@ -1,24 +1,35 @@ -#[cfg(feature = "enterprise")] +#[cfg(feature = "private")] +#[allow(unused)] +pub use crate::ee::*; + +#[cfg(all(feature = "enterprise", not(feature = "private")))] use crate::db::DB; -use crate::ee::LicensePlan::Community; -#[cfg(feature = "enterprise")] +#[cfg(not(feature = "private"))] +use crate::ee_oss::LicensePlan::Community; +#[cfg(all(feature = "enterprise", not(feature = "private")))] use crate::error; +#[cfg(not(feature = "private"))] use serde::Deserialize; +#[cfg(not(feature = "private"))] use std::sync::Arc; +#[cfg(not(feature = "private"))] use tokio::sync::RwLock; +#[cfg(not(feature = "private"))] lazy_static::lazy_static! { pub static ref LICENSE_KEY_VALID: Arc> = Arc::new(RwLock::new(true)); pub static ref LICENSE_KEY_ID: Arc> = Arc::new(RwLock::new("".to_string())); pub static ref LICENSE_KEY: Arc> = Arc::new(RwLock::new("".to_string())); } +#[cfg(not(feature = "private"))] pub enum LicensePlan { Community, Pro, Enterprise, } +#[cfg(not(feature = "private"))] pub async fn get_license_plan() -> LicensePlan { // Implementation is not open source return Community; @@ -26,6 +37,7 @@ pub async fn get_license_plan() -> LicensePlan { #[derive(Deserialize)] #[serde(untagged)] +#[cfg(not(feature = "private"))] pub enum CriticalErrorChannel { Email { email: String }, Slack { slack_channel: String }, @@ -33,6 +45,7 @@ pub enum CriticalErrorChannel { } #[derive(Deserialize)] +#[cfg(not(feature = "private"))] pub struct TeamsChannel { pub team_id: String, pub team_name: String, @@ -40,6 +53,7 @@ pub struct TeamsChannel { pub channel_name: String, } +#[cfg(not(feature = "private"))] pub enum CriticalAlertKind { #[cfg(feature = "enterprise")] CriticalError, @@ -47,7 +61,7 @@ pub enum CriticalAlertKind { RecoveredCriticalError, } -#[cfg(feature = "enterprise")] +#[cfg(all(feature = "enterprise", not(feature = "private")))] pub async fn send_critical_alert( _error_message: String, _db: &DB, @@ -56,7 +70,7 @@ pub async fn send_critical_alert( ) { } -#[cfg(feature = "enterprise")] +#[cfg(all(feature = "enterprise", not(feature = "private")))] pub async fn maybe_renew_license_key_on_start( _http_client: &reqwest::Client, _db: &crate::db::DB, @@ -66,14 +80,14 @@ pub async fn maybe_renew_license_key_on_start( force_renew_now } -#[cfg(feature = "enterprise")] +#[cfg(all(feature = "enterprise", not(feature = "private")))] pub enum RenewReason { Manual, Schedule, OnStart, } -#[cfg(feature = "enterprise")] +#[cfg(all(feature = "enterprise", not(feature = "private")))] pub async fn renew_license_key( _http_client: &reqwest::Client, _db: &crate::db::DB, @@ -84,7 +98,7 @@ pub async fn renew_license_key( "".to_string() } -#[cfg(feature = "enterprise")] +#[cfg(all(feature = "enterprise", not(feature = "private")))] pub async fn create_customer_portal_session( _http_client: &reqwest::Client, _key: Option, @@ -93,14 +107,18 @@ pub async fn create_customer_portal_session( Ok("".to_string()) } -#[cfg(feature = "enterprise")] +#[cfg(all(feature = "enterprise", not(feature = "private")))] pub async fn worker_groups_alerts(_db: &DB) {} -#[cfg(feature = "enterprise")] +#[cfg(all(feature = "enterprise", not(feature = "private")))] pub async fn jobs_waiting_alerts(_db: &DB) {} -#[cfg(feature = "enterprise")] -pub async fn low_disk_alerts(_db: &DB, _server_mode: bool, _worker_mode: bool, _workers: Vec) { +#[cfg(all(feature = "enterprise", not(feature = "private")))] +pub async fn low_disk_alerts( + _db: &DB, + _server_mode: bool, + _worker_mode: bool, + _workers: Vec, +) { // Implementation is not open source } - diff --git a/backend/windmill-common/src/email_ee.rs b/backend/windmill-common/src/email_oss.rs similarity index 61% rename from backend/windmill-common/src/email_ee.rs rename to backend/windmill-common/src/email_oss.rs index 42aebbeec3..e6a340523e 100644 --- a/backend/windmill-common/src/email_ee.rs +++ b/backend/windmill-common/src/email_oss.rs @@ -1,5 +1,11 @@ +#[cfg(feature = "private")] +#[allow(unused)] +pub use crate::email_ee::*; + +#[cfg(not(feature = "private"))] use crate::server::Smtp; +#[cfg(not(feature = "private"))] pub async fn send_email( _subject: &str, _content: &str, diff --git a/backend/windmill-common/src/job_s3_helpers_ee.rs b/backend/windmill-common/src/job_s3_helpers_oss.rs similarity index 76% rename from backend/windmill-common/src/job_s3_helpers_ee.rs rename to backend/windmill-common/src/job_s3_helpers_oss.rs index d5d6ac4052..b7ebd57f3e 100644 --- a/backend/windmill-common/src/job_s3_helpers_ee.rs +++ b/backend/windmill-common/src/job_s3_helpers_oss.rs @@ -1,5 +1,11 @@ +#[cfg(feature = "private")] +#[allow(unused)] +pub use crate::job_s3_helpers_ee::*; + +#[cfg(not(feature = "private"))] use crate::s3_helpers::{ObjectStoreResource, StorageResourceType}; +#[cfg(not(feature = "private"))] pub async fn get_s3_resource_internal<'c>( _resource_type: StorageResourceType, _s3_resource_value_raw: serde_json::Value, @@ -9,11 +15,13 @@ pub async fn get_s3_resource_internal<'c>( todo!() } +#[cfg(not(feature = "private"))] pub enum TokenGenerator<'c> { AsClient(&'c crate::client::AuthedClient), AsServerInstance(), } +#[cfg(not(feature = "private"))] impl<'c> TokenGenerator<'c> { pub async fn gen_token( &self, @@ -24,7 +32,7 @@ impl<'c> TokenGenerator<'c> { } } -#[cfg(feature = "parquet")] +#[cfg(all(feature = "parquet", not(feature = "private")))] pub(crate) async fn generate_s3_aws_oidc_resource<'c>( _clone: crate::s3_helpers::S3AwsOidcResource, _token_generator: TokenGenerator<'c>, diff --git a/backend/windmill-common/src/lib.rs b/backend/windmill-common/src/lib.rs index 6d92cd8f55..315916d0d4 100644 --- a/backend/windmill-common/src/lib.rs +++ b/backend/windmill-common/src/lib.rs @@ -19,7 +19,7 @@ use std::{ use tokio::sync::broadcast; -use ee::CriticalErrorChannel; +use ee_oss::CriticalErrorChannel; use error::Error; use scripts::ScriptLang; use sqlx::{Pool, Postgres}; @@ -32,8 +32,12 @@ pub mod bench; pub mod cache; pub mod client; pub mod db; +#[cfg(feature = "private")] pub mod ee; +pub mod ee_oss; +#[cfg(feature = "private")] pub mod email_ee; +pub mod email_oss; pub mod error; pub mod external_ip; pub mod flow_status; @@ -41,25 +45,35 @@ pub mod flows; pub mod global_settings; pub mod indexer; pub mod job_metrics; -#[cfg(feature = "parquet")] +#[cfg(all(feature = "parquet", feature = "private"))] pub mod job_s3_helpers_ee; +#[cfg(feature = "parquet")] +pub mod job_s3_helpers_oss; -#[cfg(all(feature = "enterprise", feature = "openidconnect"))] +#[cfg(all(feature = "enterprise", feature = "openidconnect", feature = "private"))] pub mod oidc_ee; +#[cfg(all(feature = "enterprise", feature = "openidconnect"))] +pub mod oidc_oss; pub mod jobs; pub mod jwt; pub mod more_serde; pub mod oauth2; +#[cfg(feature = "private")] pub mod otel_ee; +pub mod otel_oss; pub mod queue; pub mod s3_helpers; pub mod schedule; pub mod schema; pub mod scripts; pub mod server; +#[cfg(feature = "private")] pub mod stats_ee; +pub mod stats_oss; +#[cfg(feature = "private")] pub mod teams_ee; +pub mod teams_oss; pub mod tracing_init; pub mod users; pub mod utils; diff --git a/backend/windmill-common/src/oidc_ee.rs b/backend/windmill-common/src/oidc_ee.rs deleted file mode 100644 index e7a157b04d..0000000000 --- a/backend/windmill-common/src/oidc_ee.rs +++ /dev/null @@ -1,198 +0,0 @@ -/* - * Author: Ruben Fiszel - * Copyright: Windmill Labs, Inc 2023 - * This file and its contents are licensed under the AGPLv3 License. - * Please see the included NOTICE for copyright information and - * LICENSE-AGPL for a copy of the license. - */ - -use serde::{Deserialize, Serialize}; -use tokio::sync::RwLock; -#[cfg(all(feature = "enterprise", feature = "openidconnect"))] -use { - crate::db::DB, - crate::{auth::IdToken as WindmillIdToken, error::Result}, - anyhow, - openidconnect::{ - core::{CoreJwsSigningAlgorithm, CoreRsaPrivateSigningKey}, - IssuerUrl, JsonWebKeyId, - }, - std::process::Command, -}; - -#[cfg(feature = "openidconnect")] -use openidconnect::AdditionalClaims; - -#[cfg(feature = "openidconnect")] -impl AdditionalClaims for JobClaim {} - -#[cfg(feature = "openidconnect")] -impl AdditionalClaims for WorkspaceClaim {} - -#[cfg(feature = "openidconnect")] -impl AdditionalClaims for InstanceClaim {} - -#[derive(Clone, Debug, Default, Deserialize, PartialEq, Eq, Serialize)] -pub struct WorkspaceClaim { - pub workspace: String, -} - -#[derive(Clone, Debug, Default, Deserialize, PartialEq, Eq, Serialize)] -pub struct InstanceClaim {} - -#[derive(Clone, Debug, Default, Deserialize, PartialEq, Eq, Serialize)] -pub struct JobClaim { - pub job_id: String, - pub path: Option, - pub flow_path: Option, - pub groups: Vec, - pub username: String, - pub email: String, - pub workspace: String, -} - -lazy_static::lazy_static! { - static ref PRIVATE_KEY: RwLock> = RwLock::new(None); -} - -pub async fn generate_id_token( - db: Option<&DB>, - claim: T, - audience: &str, - identifier: String, - email: Option, -) -> Result { - use chrono::{Duration, Utc}; - use openidconnect::{ - core::{CoreGenderClaim, CoreJweContentEncryptionAlgorithm}, - Audience, EndUserEmail, IdToken, IdTokenClaims, StandardClaims, SubjectIdentifier, - }; - - let private_key = get_private_key(db).await?; - - let issue_url = format!("{}/api/oidc/", crate::BASE_URL.read().await.clone()); - let issue_time = Utc::now(); - let expiration = issue_time + Duration::try_hours(48).unwrap(); - let id_token = IdToken::< - T, - CoreGenderClaim, - CoreJweContentEncryptionAlgorithm, - CoreJwsSigningAlgorithm, - >::new( - IdTokenClaims::::new( - // Specify the issuer URL for the OpenID Connect Provider. - IssuerUrl::new(issue_url) - .map_err(|e| anyhow::anyhow!("Failed to generate IssueUrl: {}", e))?, - // The audience is usually a single entry with the client ID of the client for whom - // the ID token is intended. This is a required claim. - vec![Audience::new(audience.to_string())], - // The ID token expiration is usually much shorter than that of the access or refresh - // tokens issued to clients. - expiration, - // The issue time is usually the current time. - issue_time, - // Set the standard claims defined by the OpenID Connect Core spec. - StandardClaims::new( - // Stable subject identifiers are recommended in place of e-mail addresses or other - // potentially unstable identifiers. This is the only required claim. - SubjectIdentifier::new(identifier), - ) - // Optional: specify the user's e-mail address. This should only be provided if the - // client has been granted the 'profile' or 'email' scopes. - .set_email(email.map(|x| EndUserEmail::new(x))) - // Optional: specify whether the provider has verified the user's e-mail address. - .set_email_verified(Some(true)), - // OpenID Connect Providers may supply custom claims by providing a struct that - // implements the AdditionalClaims trait. This requires manually using the - // generic IdTokenClaims struct rather than the CoreIdTokenClaims type alias, - // however. - claim, - ), - // The private key used for signing the ID token. For confidential clients (those able - // to maintain a client secret), a CoreHmacKey can also be used, in conjunction - // with one of the CoreJwsSigningAlgorithm::HmacSha* signing algorithms. When using an - // HMAC-based signing algorithm, the UTF-8 representation of the client secret should - // be used as the HMAC key. - &CoreRsaPrivateSigningKey::from_pem( - &private_key, - Some(JsonWebKeyId::new("windmill".to_string())), - ) - .map_err(|e| anyhow::anyhow!("Invalid private key: {}", e))?, - // Uses the RS256 signature algorithm. This crate supports any RS*, PS*, or HS* - // signature algorithm. - CoreJwsSigningAlgorithm::RsaSsaPkcs1V15Sha256, - // When returning the ID token alongside an access token (e.g., in the Authorization Code - // flow), it is recommended to pass the access token here to set the `at_hash` claim - // automatically. - None, - // When returning the ID token alongside an authorization code (e.g., in the implicit - // flow), it is recommended to pass the authorization code here to set the `c_hash` claim - // automatically. - None, - ) - .map_err(|e| anyhow::anyhow!("Failed to generate token: {}", e))?; - - Ok(WindmillIdToken::new(id_token.to_string(), expiration)) -} - -#[cfg(all(feature = "enterprise", feature = "openidconnect"))] -pub async fn get_private_key(db: Option<&DB>) -> anyhow::Result { - if let Some(key) = PRIVATE_KEY.read().await.clone() { - return Ok(key); - } else if let Some(db) = db { - let key = sqlx::query_scalar!( - "SELECT value->>'private_key' FROM global_settings WHERE name = 'rsa_keys'", - ) - .fetch_optional(db) - .await? - .flatten(); - - let key = key.filter(|s| !s.is_empty()); - - if let Some(key) = key { - return Ok(key); - } else { - let keys = gen_pems(db).await?; - return Ok(keys.private_key); - } - } else { - return Err(anyhow::anyhow!("Private key not found and no db provided")); - } -} - -#[cfg(all(feature = "enterprise", feature = "openidconnect"))] -#[derive(Debug, Clone, serde::Serialize)] -struct Keys { - private_key: String, -} - -#[cfg(all(feature = "enterprise", feature = "openidconnect"))] -async fn gen_pems(db: &DB) -> anyhow::Result { - use anyhow::anyhow; - - let private_key_cmd = Command::new("openssl") - .arg("genrsa") - .arg("--traditional") - .arg("2048") - .output() - .expect("failed to execute process"); - - let private_key = String::from_utf8(private_key_cmd.stdout)?; - - tracing::debug!("Generated private key: {}", private_key); - - if private_key.is_empty() { - return Err(anyhow!("Failed to generate RSA key: key is empty")); - } - - let keys = Keys { private_key }; - - sqlx::query!( - r#"INSERT INTO global_settings (name, value) VALUES ('rsa_keys', $1)"#, - serde_json::to_value(&keys).unwrap() - ) - .execute(db) - .await?; - - Ok(keys) -} diff --git a/backend/windmill-common/src/oidc_oss.rs b/backend/windmill-common/src/oidc_oss.rs new file mode 100644 index 0000000000..0a42cee2ed --- /dev/null +++ b/backend/windmill-common/src/oidc_oss.rs @@ -0,0 +1,92 @@ +#[cfg(feature = "private")] +#[allow(unused)] +pub use crate::oidc_ee::*; + +/* + * Author: Ruben Fiszel + * Copyright: Windmill Labs, Inc 2023 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + +#[cfg(not(feature = "private"))] +use serde::{Deserialize, Serialize}; +#[cfg(not(feature = "private"))] +use tokio::sync::RwLock; +#[cfg(all( + feature = "enterprise", + feature = "openidconnect", + not(feature = "private") +))] +use { + crate::db::DB, + crate::{ + auth::IdToken as WindmillIdToken, + error::{Error, Result}, + }, + anyhow, +}; + +#[cfg(all(feature = "openidconnect", not(feature = "private")))] +use openidconnect::AdditionalClaims; + +#[cfg(all(feature = "openidconnect", not(feature = "private")))] +impl AdditionalClaims for JobClaim {} + +#[cfg(all(feature = "openidconnect", not(feature = "private")))] +impl AdditionalClaims for WorkspaceClaim {} + +#[cfg(all(feature = "openidconnect", not(feature = "private")))] +impl AdditionalClaims for InstanceClaim {} + +#[derive(Clone, Debug, Default, Deserialize, PartialEq, Eq, Serialize)] +#[cfg(not(feature = "private"))] +pub struct WorkspaceClaim { + pub workspace: String, +} + +#[derive(Clone, Debug, Default, Deserialize, PartialEq, Eq, Serialize)] +#[cfg(not(feature = "private"))] +pub struct InstanceClaim {} + +#[derive(Clone, Debug, Default, Deserialize, PartialEq, Eq, Serialize)] +#[cfg(not(feature = "private"))] +pub struct JobClaim { + pub job_id: String, + pub path: Option, + pub flow_path: Option, + pub groups: Vec, + pub username: String, + pub email: String, + pub workspace: String, +} + +#[cfg(not(feature = "private"))] +lazy_static::lazy_static! { + static ref PRIVATE_KEY: RwLock> = RwLock::new(None); +} + +#[cfg(not(feature = "private"))] +pub async fn generate_id_token( + _db: Option<&DB>, + _claim: T, + _audience: &str, + _identifier: String, + _email: Option, +) -> Result { + Err(Error::internal_err( + "Not implemented in Windmill's Open Source repository".to_string(), + )) +} + +#[cfg(all( + feature = "enterprise", + feature = "openidconnect", + not(feature = "private") +))] +pub async fn get_private_key(_db: Option<&DB>) -> anyhow::Result { + Err(anyhow::anyhow!( + "Not implemented in Windmill's Open Source repository" + )) +} diff --git a/backend/windmill-common/src/otel_ee.rs b/backend/windmill-common/src/otel_oss.rs similarity index 60% rename from backend/windmill-common/src/otel_ee.rs rename to backend/windmill-common/src/otel_oss.rs index f3ada162f6..9225915636 100644 --- a/backend/windmill-common/src/otel_ee.rs +++ b/backend/windmill-common/src/otel_oss.rs @@ -1,3 +1,7 @@ +#[cfg(feature = "private")] +#[allow(unused)] +pub use crate::otel_ee::*; + /* * Author: Ruben Fiszel * Copyright: Windmill Labs, Inc 2022 @@ -6,43 +10,51 @@ * LICENSE-AGPL for a copy of the license. */ +#[cfg(not(feature = "private"))] use crate::{jobs::QueuedJob, utils::Mode}; +#[cfg(not(feature = "private"))] use uuid::Uuid; +#[cfg(not(feature = "private"))] pub fn set_span_parent(_span: &tracing::Span, _rj: &Uuid) {} -#[cfg(not(all(feature = "otel", feature = "enterprise")))] +#[cfg(all( + not(all(feature = "otel", feature = "enterprise")), + not(feature = "private") +))] pub(crate) type OtelProvider = Option<()>; -#[cfg(all(feature = "otel", feature = "enterprise"))] +#[cfg(all(feature = "otel", feature = "enterprise", not(feature = "private")))] pub(crate) type OtelProvider = Option; -#[cfg(not(feature = "otel"))] +#[cfg(all(not(feature = "otel"), not(feature = "private")))] pub fn otel_ctx() -> () {} -#[cfg(feature = "otel")] +#[cfg(all(feature = "otel", not(feature = "private")))] #[inline(always)] pub fn otel_ctx() -> opentelemetry::Context { opentelemetry::Context::current() } -#[cfg(not(feature = "otel"))] +#[cfg(all(not(feature = "otel"), not(feature = "private")))] impl FutureExt for T {} -#[cfg(not(feature = "otel"))] +#[cfg(all(not(feature = "otel"), not(feature = "private")))] pub trait FutureExt: Sized { fn with_context(self, _otel_cx: ()) -> Self { self } } +#[cfg(not(feature = "private"))] use tracing_subscriber::EnvFilter; +#[cfg(not(feature = "private"))] pub(crate) fn init_logs_bridge(_mode: &Mode, _hostname: &str, _env: &str) -> Option { None } -#[cfg(all(feature = "otel", feature = "enterprise"))] +#[cfg(all(feature = "otel", feature = "enterprise", not(feature = "private")))] pub(crate) fn init_otlp_tracer( _mode: &Mode, _hostname: &str, @@ -51,8 +63,10 @@ pub(crate) fn init_otlp_tracer( None } +#[cfg(not(feature = "private"))] pub(crate) fn init_meter_provider(_mode: &Mode, _hostname: &str, _env: &str) -> OtelProvider { None } +#[cfg(not(feature = "private"))] pub fn add_root_flow_job_to_otlp(_queued_job: &QueuedJob, _success: bool) {} diff --git a/backend/windmill-common/src/s3_helpers.rs b/backend/windmill-common/src/s3_helpers.rs index 7e7c48851e..171958f5cd 100644 --- a/backend/windmill-common/src/s3_helpers.rs +++ b/backend/windmill-common/src/s3_helpers.rs @@ -147,7 +147,7 @@ pub enum ObjectStoreReload { #[cfg(feature = "parquet")] pub async fn reload_object_store_setting(db: &crate::DB) -> ObjectStoreReload { use crate::{ - ee::{get_license_plan, LicensePlan}, + ee_oss::{get_license_plan, LicensePlan}, global_settings::{load_value_from_global_settings, OBJECT_STORE_CONFIG_SETTING}, s3_helpers::ObjectSettings, }; @@ -613,8 +613,8 @@ pub async fn build_object_store_from_settings( build_azure_blob_client(&azure_blob_resource).map(|x| ExpirableObjectStore::from(x)) } ObjectSettings::AwsOidc(ref s3_aws_oidc_settings) => { - let token_generator = crate::job_s3_helpers_ee::TokenGenerator::AsServerInstance(); - let res = crate::job_s3_helpers_ee::generate_s3_aws_oidc_resource( + let token_generator = crate::job_s3_helpers_oss::TokenGenerator::AsServerInstance(); + let res = crate::job_s3_helpers_oss::generate_s3_aws_oidc_resource( s3_aws_oidc_settings.clone(), token_generator, init_private_key, diff --git a/backend/windmill-common/src/scripts.rs b/backend/windmill-common/src/scripts.rs index dda9daf0f4..ffa7bd486a 100644 --- a/backend/windmill-common/src/scripts.rs +++ b/backend/windmill-common/src/scripts.rs @@ -9,6 +9,7 @@ use std::{ fmt::{self, Display}, hash::{Hash, Hasher}, + str::FromStr, }; use crate::{ @@ -21,6 +22,7 @@ use crate::worker::HUB_CACHE_DIR; use anyhow::Context; use backon::ConstantBuilder; use backon::{BackoffBuilder, Retryable}; +use itertools::Itertools; use serde::de::Error as _; use serde::{ser::SerializeSeq, Deserialize, Deserializer, Serialize}; @@ -85,6 +87,40 @@ impl ScriptLang { } } +impl FromStr for ScriptLang { + type Err = Error; + fn from_str(s: &str) -> Result { + let language = match s.to_lowercase().as_str() { + "bun" => ScriptLang::Bun, + "bunnative" => ScriptLang::Bunnative, + "nativets" => ScriptLang::Nativets, + "deno" => ScriptLang::Deno, + "python3" => ScriptLang::Python3, + "go" => ScriptLang::Go, + "bash" => ScriptLang::Bash, + "powershell" => ScriptLang::Powershell, + "postgresql" => ScriptLang::Postgresql, + "mysql" => ScriptLang::Mysql, + "bigquery" => ScriptLang::Bigquery, + "snowflake" => ScriptLang::Snowflake, + "mssql" => ScriptLang::Mssql, + "graphql" => ScriptLang::Graphql, + "oracledb" => ScriptLang::OracleDB, + "php" => ScriptLang::Php, + "rust" => ScriptLang::Rust, + "ansible" => ScriptLang::Ansible, + "csharp" => ScriptLang::CSharp, + "nu" => ScriptLang::Nu, + "java" => ScriptLang::Java, + language => { + return Err(anyhow::anyhow!("{} is currently not supported", language).into()) + } + }; + + Ok(language) + } +} + #[derive(Eq, PartialEq, Debug, Hash, Clone, Copy, sqlx::Type)] #[sqlx(transparent)] pub struct ScriptHash(pub i64); @@ -367,7 +403,7 @@ where deserializer.deserialize_any(StringOrArrayVisitor) } -#[derive(Deserialize)] +#[derive(Debug, Deserialize)] pub struct ListScriptQuery { pub path_start: Option, pub path_exact: Option, @@ -384,6 +420,29 @@ pub struct ListScriptQuery { pub include_without_main: Option, pub include_draft_only: Option, pub with_deployment_msg: Option, + #[serde(default, deserialize_with = "from_seq")] + pub languages: Option>, +} + +fn from_seq<'de, D>(deserializer: D) -> Result>, D::Error> +where + D: Deserializer<'de>, +{ + let s = ::deserialize(deserializer)?; + + let languages: Vec = s + .split(",") + .map(ScriptLang::from_str) + .try_collect() + .map_err(|e| serde::de::Error::custom(e.to_string()))?; + + let languages = if languages.is_empty() { + None + } else { + Some(languages) + }; + + Ok(languages) } pub fn to_i64(s: &str) -> crate::error::Result { diff --git a/backend/windmill-common/src/stats_ee.rs b/backend/windmill-common/src/stats_oss.rs similarity index 73% rename from backend/windmill-common/src/stats_ee.rs rename to backend/windmill-common/src/stats_oss.rs index 5d2dc82b82..829e6e3a5b 100644 --- a/backend/windmill-common/src/stats_ee.rs +++ b/backend/windmill-common/src/stats_oss.rs @@ -1,17 +1,26 @@ +#[cfg(feature = "private")] +#[allow(unused)] +pub use crate::stats_ee::*; + +#[cfg(not(feature = "private"))] use sqlx::Postgres; +#[cfg(not(feature = "private"))] use crate::{error::Result, scripts::ScriptLang, DB}; +#[cfg(not(feature = "private"))] pub async fn get_disable_stats_setting(_db: &DB) -> bool { // stats details are closed source false } +#[cfg(not(feature = "private"))] pub async fn schedule_stats(_db: &DB, _http_client: &reqwest::Client) -> () { // stats details are closed source } +#[cfg(not(feature = "private"))] #[derive(Debug, sqlx::FromRow, serde::Serialize)] struct JobsUsage { language: Option, @@ -19,12 +28,14 @@ struct JobsUsage { count: i64, } +#[cfg(not(feature = "private"))] pub enum SendStatsReason { Manual, Schedule, OnStart, } +#[cfg(not(feature = "private"))] pub async fn send_stats( _http_client: &reqwest::Client, _db: &DB, @@ -34,11 +45,13 @@ pub async fn send_stats( Ok(()) } +#[cfg(not(feature = "private"))] pub struct ActiveUserUsage { pub author_count: Option, pub operator_count: Option, } +#[cfg(not(feature = "private"))] pub async fn get_user_usage<'c, E: sqlx::Executor<'c, Database = Postgres>>( _db: E, ) -> Result { diff --git a/backend/windmill-common/src/teams_ee.rs b/backend/windmill-common/src/teams_ee.rs deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/backend/windmill-common/src/teams_oss.rs b/backend/windmill-common/src/teams_oss.rs new file mode 100644 index 0000000000..347ad8e6d7 --- /dev/null +++ b/backend/windmill-common/src/teams_oss.rs @@ -0,0 +1,5 @@ +#[cfg(feature = "private")] +#[allow(unused)] +pub use crate::teams_ee::*; + + diff --git a/backend/windmill-common/src/tracing_init.rs b/backend/windmill-common/src/tracing_init.rs index 5ed5cec752..77194ac0a9 100644 --- a/backend/windmill-common/src/tracing_init.rs +++ b/backend/windmill-common/src/tracing_init.rs @@ -48,7 +48,7 @@ pub fn initialize_tracing( hostname: &str, mode: &Mode, environment: &str, -) -> (WorkerGuard, crate::otel_ee::OtelProvider) { +) -> (WorkerGuard, crate::otel_oss::OtelProvider) { let style = std::env::var("RUST_LOG_STYLE").unwrap_or_else(|_| "auto".into()); let rust_log_env = std::env::var("RUST_LOG"); @@ -70,16 +70,16 @@ pub fn initialize_tracing( LevelFilter::INFO }; - let meter_provider = crate::otel_ee::init_meter_provider(mode, hostname, environment); + let meter_provider = crate::otel_oss::init_meter_provider(mode, hostname, environment); #[cfg(all(feature = "otel", feature = "enterprise"))] - let opentelemetry = crate::otel_ee::init_otlp_tracer(mode, hostname, environment) + let opentelemetry = crate::otel_oss::init_otlp_tracer(mode, hostname, environment) .map(|x| tracing_opentelemetry::layer().with_tracer(x)); #[cfg(not(all(feature = "otel", feature = "enterprise")))] let opentelemetry: Option = None; - let logs_bridge = crate::otel_ee::init_logs_bridge(&mode, hostname, environment); + let logs_bridge = crate::otel_oss::init_logs_bridge(&mode, hostname, environment); use tracing_appender::rolling::{RollingFileAppender, Rotation}; diff --git a/backend/windmill-common/src/utils.rs b/backend/windmill-common/src/utils.rs index 3f1a1bd385..7c2faea868 100644 --- a/backend/windmill-common/src/utils.rs +++ b/backend/windmill-common/src/utils.rs @@ -7,9 +7,9 @@ */ use crate::auth::is_devops_email; -use crate::ee::LICENSE_KEY_ID; +use crate::ee_oss::LICENSE_KEY_ID; #[cfg(feature = "enterprise")] -use crate::ee::{send_critical_alert, CriticalAlertKind}; +use crate::ee_oss::{send_critical_alert, CriticalAlertKind}; use crate::error::{to_anyhow, Error, Result}; use crate::global_settings::UNIQUE_ID_SETTING; use crate::DB; @@ -25,7 +25,8 @@ use semver::Version; use serde::{Deserialize, Deserializer, Serialize}; use sha2::{Digest, Sha256}; use sqlx::{Pool, Postgres}; -use std::str::FromStr; +use std::{fs::DirBuilder as SyncDirBuilder, str::FromStr}; +use tokio::fs::DirBuilder as AsyncDirBuilder; pub const MAX_PER_PAGE: usize = 10000; pub const DEFAULT_PER_PAGE: usize = 1000; @@ -33,6 +34,10 @@ pub const DEFAULT_PER_PAGE: usize = 1000; pub const GIT_VERSION: &str = git_version!(args = ["--tag", "--always"], fallback = "unknown-version"); +pub const AGENT_JWT_PREFIX: &str = "jwt_agent_"; +pub const WORKER_NAME_PREFIX: &str = "wk"; +pub const AGENT_WORKER_NAME_PREFIX: &str = "ag"; + use crate::CRITICAL_ALERT_MUTE_UI_ENABLED; use std::panic::{self, AssertUnwindSafe, Location}; use std::sync::atomic::Ordering; @@ -53,6 +58,12 @@ lazy_static::lazy_static! { } ).unwrap_or(Version::new(0, 1, 0)); + pub static ref HOSTNAME :String = std::env::var("FORCE_HOSTNAME").unwrap_or_else(|_| { + gethostname() + .to_str() + .map(|x| x.to_string()) + .unwrap_or_else(|| rd_string(5)) + }); pub static ref MODE_AND_ADDONS: ModeAndAddons = { let mut search_addon = false; @@ -120,6 +131,10 @@ lazy_static::lazy_static! { }; } +lazy_static::lazy_static! { + pub static ref AGENT_TOKEN: String = std::env::var("AGENT_TOKEN").unwrap_or_default(); +} + #[derive(Clone)] pub struct ModeAndAddons { pub indexer: bool, @@ -167,15 +182,6 @@ pub async fn require_admin_or_devops( Ok(()) } -pub fn hostname() -> String { - std::env::var("FORCE_HOSTNAME").unwrap_or_else(|_| { - gethostname() - .to_str() - .map(|x| x.to_string()) - .unwrap_or_else(|| rd_string(5)) - }) -} - fn instance_name(hostname: &str) -> String { hostname .replace(" ", "") @@ -186,15 +192,30 @@ fn instance_name(hostname: &str) -> String { .to_string() } -pub fn worker_suffix(hostname: &str, rd_string: &str) -> String { - format!("{}-{}", instance_name(hostname), rd_string) +const DEFAULT_WORKER_SUFFIX_LEN: usize = 5; +pub const SSH_AGENT_WORKER_SUFFIX: &'static str = "/ssh"; + +pub fn create_worker_suffix(hostname: &str, rd_string_len: usize, ssh_ag_worker: bool) -> String { + let mut wk_suffix = format!("{}-{}", instance_name(hostname), rd_string(rd_string_len)); + if ssh_ag_worker { + wk_suffix.push_str(SSH_AGENT_WORKER_SUFFIX); + } + wk_suffix +} + +pub fn create_ssh_agent_worker_suffix(hostname: &str) -> String { + create_worker_suffix(hostname, DEFAULT_WORKER_SUFFIX_LEN, true) +} + +pub fn create_default_worker_suffix(hostname: &str) -> String { + create_worker_suffix(hostname, DEFAULT_WORKER_SUFFIX_LEN, false) } pub fn worker_name_with_suffix(is_agent: bool, worker_group: &str, suffix: &str) -> String { if is_agent { - format!("ag-{}-{}", worker_group, suffix) + format!("{}-{}-{}", AGENT_WORKER_NAME_PREFIX, worker_group, suffix) } else { - format!("wk-{}-{}", worker_group, suffix) + format!("{}-{}-{}", WORKER_NAME_PREFIX, worker_group, suffix) } } @@ -223,6 +244,21 @@ pub async fn now_from_db<'c, E: sqlx::PgExecutor<'c>>( .unwrap()) } +pub async fn create_directory_async(directory_path: &str) { + AsyncDirBuilder::new() + .recursive(true) + .create(directory_path) + .await + .expect("could not create dir"); +} + +pub fn create_directory_sync(directory_path: &str) { + SyncDirBuilder::new() + .recursive(true) + .create(directory_path) + .expect("could not create dir"); +} + pub fn not_found_if_none>(opt: Option, kind: &str, name: U) -> Result { if let Some(o) = opt { Ok(o) diff --git a/backend/windmill-common/src/worker.rs b/backend/windmill-common/src/worker.rs index 6b11fa086f..1f09176512 100644 --- a/backend/windmill-common/src/worker.rs +++ b/backend/windmill-common/src/worker.rs @@ -1,4 +1,5 @@ use anyhow::anyhow; +use axum::http::HeaderMap; use bytes::Bytes; use const_format::concatcp; use itertools::Itertools; @@ -34,7 +35,6 @@ use crate::{ pub const DEFAULT_CLOUD_TIMEOUT: u64 = 900; pub const DEFAULT_SELFHOSTED_TIMEOUT: u64 = 604800; // 7 days - lazy_static::lazy_static! { pub static ref WORKER_GROUP: String = std::env::var("WORKER_GROUP").unwrap_or_else(|_| { #[cfg(not(feature = "enterprise"))] @@ -129,7 +129,7 @@ lazy_static::lazy_static! { pub static ref ALL_TAGS: Arc>> = Arc::new(RwLock::new(vec![])); - static ref CUSTOM_TAG_REGEX: Regex = Regex::new(r"^([\w-]+)\(((?:[\w-])+\+?)+\)$").unwrap(); + static ref CUSTOM_TAG_REGEX: Regex = Regex::new(r"^([\w-]+)\(((?:[\w-]+\+)*[\w-]+)\)$").unwrap(); pub static ref DISABLE_BUNDLING: bool = std::env::var("DISABLE_BUNDLING") .ok() @@ -159,12 +159,20 @@ impl HttpClient { pub async fn post( &self, url: &str, + headers: Option, body: &T, ) -> anyhow::Result { - let response = self + let response_builder = self .0 .post(format!("{}{}", *BASE_INTERNAL_URL, url)) - .json(body) + .json(body); + + let response_builder = match headers { + Some(headers) => response_builder.headers(headers), + None => response_builder, + }; + + let response = response_builder .send() .await .map_err(|e| anyhow::anyhow!(e))?; @@ -259,7 +267,7 @@ fn format_pull_query(peek: String) -> String { id, workspace_id, parent_job, created_by, created_at, runnable_id, runnable_path, args, kind, trigger, trigger_kind, permissioned_as, permissioned_as_email, script_lang, - flow_innermost_root_job, flow_step_id, + flow_innermost_root_job, flow_step_id, same_worker, pre_run_error, visible_to_owner, tag, concurrent_limit, concurrency_time_window_s, timeout, cache_ttl, priority, raw_code, raw_lock, raw_flow, script_entrypoint_override, preprocessed @@ -267,13 +275,13 @@ fn format_pull_query(peek: String) -> String { WHERE id = (SELECT id FROM peek) ) SELECT j.id, j.workspace_id, j.parent_job, j.created_by, started_at, scheduled_for, j.runnable_id, j.runnable_path, j.args, canceled_by, - canceled_reason, j.kind, j.trigger, j.trigger_kind, j.permissioned_as, + canceled_reason, j.kind, j.trigger, j.trigger_kind, j.permissioned_as, flow_status, j.script_lang, - j.same_worker, j.pre_run_error, j.visible_to_owner, + j.same_worker, j.pre_run_error, j.visible_to_owner, j.tag, j.concurrent_limit, j.concurrency_time_window_s, j.flow_innermost_root_job, j.timeout, j.flow_step_id, j.cache_ttl, j.priority, j.raw_code, j.raw_lock, j.raw_flow, j.script_entrypoint_override, j.preprocessed, pj.runnable_path as parent_runnable_path, - COALESCE(p.email, j.permissioned_as_email) as permissioned_as_email, p.username as permissioned_as_username, p.is_admin as permissioned_as_is_admin, + COALESCE(p.email, j.permissioned_as_email) as permissioned_as_email, p.username as permissioned_as_username, p.is_admin as permissioned_as_is_admin, p.is_operator as permissioned_as_is_operator, p.groups as permissioned_as_groups, p.folders as permissioned_as_folders FROM q, j LEFT JOIN v2_job_status f USING (id) @@ -308,7 +316,7 @@ pub async fn store_suspended_pull_query(wc: &WorkerConfig) { } pub fn make_pull_query(tags: &[String]) -> String { - format_pull_query(format!( + let query = format_pull_query(format!( "SELECT id FROM v2_job_queue WHERE running = false AND tag IN ({}) AND scheduled_for <= now() @@ -316,7 +324,8 @@ pub fn make_pull_query(tags: &[String]) -> String { FOR UPDATE SKIP LOCKED LIMIT 1", tags.iter().map(|x| format!("'{x}'")).join(", ") - )) + )); + query } pub async fn store_pull_query(wc: &WorkerConfig) { @@ -1028,7 +1037,7 @@ pub async fn update_ping_http( Ok(()) } -#[derive(Serialize, Deserialize)] +#[derive(Debug, Serialize, Deserialize)] pub struct JobCancelled { pub canceled_by: String, pub reason: String, @@ -1060,11 +1069,11 @@ pub async fn update_ping_for_failed_init_script_query( db: &DB, ) -> anyhow::Result<()> { sqlx::query!( - "UPDATE worker_ping SET -ping_at = now(), -jobs_executed = 1, -current_job_id = $1, -current_job_workspace_id = 'admins' + "UPDATE worker_ping SET +ping_at = now(), +jobs_executed = 1, +current_job_id = $1, +current_job_workspace_id = 'admins' WHERE worker = $2", last_job_id, worker_name @@ -1510,7 +1519,7 @@ pub struct WorkerConfig { impl std::fmt::Debug for WorkerConfig { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "WorkerConfig {{ worker_tags: {:?}, priority_tags_sorted: {:?}, dedicated_worker: {:?}, init_bash: {:?}, cache_clear: {:?}, additional_python_paths: {:?}, pip_local_dependencies: {:?}, env_vars: {:?} }}", + write!(f, "WorkerConfig {{ worker_tags: {:?}, priority_tags_sorted: {:?}, dedicated_worker: {:?}, init_bash: {:?}, cache_clear: {:?}, additional_python_paths: {:?}, pip_local_dependencies: {:?}, env_vars: {:?} }}", self.worker_tags, self.priority_tags_sorted, self.dedicated_worker, self.init_bash, self.cache_clear, self.additional_python_paths, self.pip_local_dependencies, self.env_vars.iter().map(|(k, v)| format!("{}: {}{} ({} chars)", k, &v[..3.min(v.len())], "***", v.len())).collect::>().join(", ")) } } diff --git a/backend/windmill-git-sync/Cargo.toml b/backend/windmill-git-sync/Cargo.toml index d5a8412fae..6a0499639a 100644 --- a/backend/windmill-git-sync/Cargo.toml +++ b/backend/windmill-git-sync/Cargo.toml @@ -9,6 +9,7 @@ name = "windmill_git_sync" path = "./src/lib.rs" [features] +private = [] enterprise = ["windmill-queue/enterprise", "windmill-common/enterprise"] default = [] diff --git a/backend/windmill-git-sync/src/git_sync_ee.rs b/backend/windmill-git-sync/src/git_sync_oss.rs similarity index 69% rename from backend/windmill-git-sync/src/git_sync_ee.rs rename to backend/windmill-git-sync/src/git_sync_oss.rs index cc245d3d0c..70e2b53640 100644 --- a/backend/windmill-git-sync/src/git_sync_ee.rs +++ b/backend/windmill-git-sync/src/git_sync_oss.rs @@ -1,7 +1,14 @@ +#[cfg(feature = "private")] +#[allow(unused)] +pub use crate::git_sync_ee::*; + +#[cfg(not(feature = "private"))] use windmill_common::error::Result; +#[cfg(not(feature = "private"))] use crate::{DeployedObject, DB}; +#[cfg(not(feature = "private"))] pub async fn handle_deployment_metadata<'c>( _email: &str, _created_by: &str, diff --git a/backend/windmill-git-sync/src/lib.rs b/backend/windmill-git-sync/src/lib.rs index f1e0ed6ee3..d4e4ebf71f 100644 --- a/backend/windmill-git-sync/src/lib.rs +++ b/backend/windmill-git-sync/src/lib.rs @@ -10,9 +10,11 @@ use sqlx::{Pool, Postgres}; use windmill_common::scripts::ScriptHash; +#[cfg(feature = "private")] pub mod git_sync_ee; +pub mod git_sync_oss; -pub use git_sync_ee::handle_deployment_metadata; +pub use git_sync_oss::handle_deployment_metadata; pub type DB = Pool; #[derive(Clone, Debug)] diff --git a/backend/windmill-indexer/Cargo.toml b/backend/windmill-indexer/Cargo.toml index 14ab53beef..ea95d6237d 100644 --- a/backend/windmill-indexer/Cargo.toml +++ b/backend/windmill-indexer/Cargo.toml @@ -11,6 +11,7 @@ path = "src/lib.rs" [features] default = [] parquet = ["dep:object_store"] +private = [] enterprise = [] [dependencies] diff --git a/backend/windmill-indexer/src/completed_runs_ee.rs b/backend/windmill-indexer/src/completed_runs_oss.rs similarity index 66% rename from backend/windmill-indexer/src/completed_runs_ee.rs rename to backend/windmill-indexer/src/completed_runs_oss.rs index f5c6c98cf8..083f8744fa 100644 --- a/backend/windmill-indexer/src/completed_runs_ee.rs +++ b/backend/windmill-indexer/src/completed_runs_oss.rs @@ -1,17 +1,28 @@ +#[cfg(feature = "private")] +#[allow(unused)] +pub use crate::completed_runs_ee::*; + +#[cfg(not(feature = "private"))] use anyhow::anyhow; +#[cfg(not(feature = "private"))] use sqlx::{Pool, Postgres}; +#[cfg(not(feature = "private"))] use windmill_common::error::Error; +#[cfg(not(feature = "private"))] #[derive(Clone)] pub struct IndexReader; +#[cfg(not(feature = "private"))] #[derive(Clone)] pub struct IndexWriter; +#[cfg(not(feature = "private"))] pub async fn init_index(_db: &Pool) -> Result<(IndexReader, IndexWriter), Error> { Err(anyhow!("Cannot initialize index: not in EE").into()) } +#[cfg(not(feature = "private"))] pub async fn run_indexer( _db: Pool, mut _index_writer: IndexWriter, diff --git a/backend/windmill-indexer/src/indexer_ee.rs b/backend/windmill-indexer/src/indexer_ee.rs deleted file mode 100644 index 8b13789179..0000000000 --- a/backend/windmill-indexer/src/indexer_ee.rs +++ /dev/null @@ -1 +0,0 @@ - diff --git a/backend/windmill-indexer/src/indexer_oss.rs b/backend/windmill-indexer/src/indexer_oss.rs new file mode 100644 index 0000000000..198c14e526 --- /dev/null +++ b/backend/windmill-indexer/src/indexer_oss.rs @@ -0,0 +1,5 @@ +#[cfg(feature = "private")] +#[allow(unused)] +pub use crate::indexer_ee::*; + + diff --git a/backend/windmill-indexer/src/lib.rs b/backend/windmill-indexer/src/lib.rs index 59c6a627f7..6c13b551d1 100644 --- a/backend/windmill-indexer/src/lib.rs +++ b/backend/windmill-indexer/src/lib.rs @@ -1,3 +1,9 @@ +#[cfg(feature = "private")] pub mod completed_runs_ee; +pub mod completed_runs_oss; +#[cfg(feature = "private")] pub mod indexer_ee; +pub mod indexer_oss; +#[cfg(feature = "private")] pub mod service_logs_ee; +pub mod service_logs_oss; diff --git a/backend/windmill-indexer/src/service_logs_ee.rs b/backend/windmill-indexer/src/service_logs_oss.rs similarity index 68% rename from backend/windmill-indexer/src/service_logs_ee.rs rename to backend/windmill-indexer/src/service_logs_oss.rs index e04575d3f5..b6e9017f88 100644 --- a/backend/windmill-indexer/src/service_logs_ee.rs +++ b/backend/windmill-indexer/src/service_logs_oss.rs @@ -1,13 +1,24 @@ +#[cfg(feature = "private")] +#[allow(unused)] +pub use crate::service_logs_ee::*; + +#[cfg(not(feature = "private"))] use anyhow::anyhow; +#[cfg(not(feature = "private"))] use sqlx::{Pool, Postgres}; +#[cfg(not(feature = "private"))] use windmill_common::error::Error; +#[cfg(not(feature = "private"))] use windmill_common::KillpillSender; #[derive(Clone)] +#[cfg(not(feature = "private"))] pub struct ServiceLogIndexReader; #[derive(Clone)] +#[cfg(not(feature = "private"))] pub struct ServiceLogIndexWriter; +#[cfg(not(feature = "private"))] pub async fn init_index( _db: &Pool, mut _killpill_tx: KillpillSender, @@ -15,6 +26,7 @@ pub async fn init_index( Err(anyhow!("Cannot initialize index: not in EE").into()) } +#[cfg(not(feature = "private"))] pub async fn run_indexer( _db: Pool, mut _index_writer: ServiceLogIndexWriter, diff --git a/backend/windmill-queue/Cargo.toml b/backend/windmill-queue/Cargo.toml index a90fa6ccf7..4458e8aa99 100644 --- a/backend/windmill-queue/Cargo.toml +++ b/backend/windmill-queue/Cargo.toml @@ -10,6 +10,7 @@ path = "src/lib.rs" [features] default = [] +private = [] enterprise = ["windmill-common/enterprise"] cloud = [] benchmark = ["windmill-common/benchmark"] diff --git a/backend/windmill-queue/src/jobs.rs b/backend/windmill-queue/src/jobs.rs index de64071c42..3b0143c826 100644 --- a/backend/windmill-queue/src/jobs.rs +++ b/backend/windmill-queue/src/jobs.rs @@ -25,7 +25,7 @@ use sqlx::{types::Json, FromRow, Pool, Postgres, Transaction}; use tokio::{sync::RwLock, time::sleep}; use ulid::Ulid; use uuid::Uuid; -use windmill_audit::audit_ee::{audit_log, AuditAuthor}; +use windmill_audit::audit_oss::{audit_log, AuditAuthor}; use windmill_audit::ActionKind; #[cfg(feature = "benchmark")] @@ -71,7 +71,7 @@ use windmill_common::BASE_URL; use windmill_common::users::SUPERADMIN_SYNC_EMAIL; use crate::flow_status::{update_flow_status_in_progress, update_workflow_as_code_status}; -use crate::jobs_ee::update_concurrency_counter; +use crate::jobs_oss::update_concurrency_counter; use crate::schedule::{get_schedule_opt, push_scheduled_job}; use crate::tags::per_workspace_tag; @@ -391,6 +391,7 @@ pub async fn append_logs( if let Err(e) = client .post::<_, String>( &format!("/api/w/{}/agent_workers/push_logs/{}", workspace.as_ref(), job_id), + None, &logs.as_ref(), ) .await { @@ -1010,33 +1011,43 @@ pub async fn add_completed_job( #[cfg(feature = "cloud")] if *CLOUD_HOSTED && !queued_job.is_flow() && _duration > 1000 { - let additional_usage = _duration / 1000; - let w_id = &queued_job.workspace_id; - let premium_workspace = windmill_common::workspaces::is_premium_workspace(db, w_id).await; - let _ = sqlx::query!( - "INSERT INTO usage (id, is_workspace, month_, usage) - VALUES ($1, TRUE, EXTRACT(YEAR FROM current_date) * 12 + EXTRACT(MONTH FROM current_date), $2) - ON CONFLICT (id, is_workspace, month_) DO UPDATE SET usage = usage.usage + $2", - w_id, - additional_usage as i32 - ) - .execute(db) - .await - .map_err(|e| Error::internal_err(format!("updating usage: {e:#}"))); - - if !premium_workspace { + let db = db.clone(); + let w_id = queued_job.workspace_id.clone(); + let email = queued_job.permissioned_as_email.clone(); + let w_id2 = w_id.clone(); + let email2 = email.clone(); + tokio::task::spawn(async move { + let additional_usage = _duration / 1000; + let premium_workspace = windmill_common::workspaces::is_premium_workspace(&db, &w_id).await; + tokio::time::timeout(std::time::Duration::from_secs(10), async move { let _ = sqlx::query!( "INSERT INTO usage (id, is_workspace, month_, usage) - VALUES ($1, FALSE, EXTRACT(YEAR FROM current_date) * 12 + EXTRACT(MONTH FROM current_date), $2) + VALUES ($1, TRUE, EXTRACT(YEAR FROM current_date) * 12 + EXTRACT(MONTH FROM current_date), $2) ON CONFLICT (id, is_workspace, month_) DO UPDATE SET usage = usage.usage + $2", - queued_job.permissioned_as_email, + w_id, additional_usage as i32 ) - .execute(db) + .execute(&db) .await .map_err(|e| Error::internal_err(format!("updating usage: {e:#}"))); - } + + if !premium_workspace { + let _ = sqlx::query!( + "INSERT INTO usage (id, is_workspace, month_, usage) + VALUES ($1, FALSE, EXTRACT(YEAR FROM current_date) * 12 + EXTRACT(MONTH FROM current_date), $2) + ON CONFLICT (id, is_workspace, month_) DO UPDATE SET usage = usage.usage + $2", + email, + additional_usage as i32 + ) + .execute(&db) + .await + .map_err(|e| Error::internal_err(format!("updating usage: {e:#}"))); + }}).await.unwrap_or_else(|_| { + tracing::error!("Could not update usage for workspace {w_id2} and permissioned as {email2}, stopped after 10s"); + }); + }); } + #[cfg(feature = "enterprise")] if !success { @@ -2127,7 +2138,7 @@ pub struct PulledJob { // NOTE: // Precomputed by the server // Used to offload work from agent workers to server -#[derive(Serialize, Deserialize)] +#[derive(Debug, Serialize, Deserialize)] pub enum PrecomputedAgentInfo { Bun { local: String, remote: String }, Python { @@ -2138,7 +2149,7 @@ pub enum PrecomputedAgentInfo { requirements: Option }, } -#[derive(Serialize, Deserialize)] +#[derive(Debug, Serialize, Deserialize)] pub struct JobAndPerms { pub job: MiniPulledJob, pub raw_code: Option, @@ -2290,16 +2301,20 @@ pub async fn pull( db: &Pool, suspend_first: bool, worker_name: &str, - query_o: Option<(String, String)>, + query_o: Option<&(String, String)>, #[cfg(feature = "benchmark")] bench: &mut BenchmarkIter, ) -> windmill_common::error::Result { loop { - if let Some((query_suspended, query_no_suspend)) = query_o.as_ref() { + if let Some((query_suspended, query_no_suspend)) = query_o { let njob = { - let job = sqlx::query_as::<_, PulledJob>(query_suspended) + let job = if query_suspended.is_empty() { + None + } else { + sqlx::query_as::<_, PulledJob>(query_suspended) .bind(worker_name) .fetch_optional(db) - .await?; + .await? + }; if let Some(job) = job { PulledJobResult { job: Some(job), suspended: true } } else { @@ -2549,7 +2564,6 @@ async fn pull_single_job_and_mark_as_running_no_concurrency_limit<'c>( tokio::time::sleep(tokio::time::Duration::from_secs(1)).await; return Ok((None, false)); } - let r = if suspend_first { // tracing::info!("Pulling job with query: {}", query); sqlx::query_as::<_, PulledJob>(&query) @@ -3265,35 +3279,40 @@ pub async fn push<'c, 'd>( job_payload, JobPayload::Flow { .. } | JobPayload::RawFlow { .. } ) { - let workspace_usage = sqlx::query_scalar!( - "INSERT INTO usage (id, is_workspace, month_, usage) - VALUES ($1, TRUE, EXTRACT(YEAR FROM current_date) * 12 + EXTRACT(MONTH FROM current_date), 1) - ON CONFLICT (id, is_workspace, month_) DO UPDATE SET usage = usage.usage + 1 - RETURNING usage.usage", - workspace_id - ) - .fetch_one(_db) - .await - .map_err(|e| Error::internal_err(format!("updating usage: {e:#}")))?; + tokio::time::timeout(std::time::Duration::from_secs(10), async move { + let workspace_usage = sqlx::query_scalar!( + "INSERT INTO usage (id, is_workspace, month_, usage) + VALUES ($1, TRUE, EXTRACT(YEAR FROM current_date) * 12 + EXTRACT(MONTH FROM current_date), 1) + ON CONFLICT (id, is_workspace, month_) DO UPDATE SET usage = usage.usage + 1 + RETURNING usage.usage", + workspace_id + ) + .fetch_one(_db) + .await + .map_err(|e| Error::internal_err(format!("updating usage: {e:#}")))?; - let user_usage = if !premium_workspace { - Some(sqlx::query_scalar!( - "INSERT INTO usage (id, is_workspace, month_, usage) - VALUES ($1, FALSE, EXTRACT(YEAR FROM current_date) * 12 + EXTRACT(MONTH FROM current_date), 1) - ON CONFLICT (id, is_workspace, month_) DO UPDATE SET usage = usage.usage + 1 - RETURNING usage.usage", - email - ) - .fetch_one(_db) - .await - .map_err(|e| Error::internal_err(format!("updating usage: {e:#}")))?) - } else { - None - }; - (Some(workspace_usage), user_usage) + let user_usage = if !premium_workspace { + Some(sqlx::query_scalar!( + "INSERT INTO usage (id, is_workspace, month_, usage) + VALUES ($1, FALSE, EXTRACT(YEAR FROM current_date) * 12 + EXTRACT(MONTH FROM current_date), 1) + ON CONFLICT (id, is_workspace, month_) DO UPDATE SET usage = usage.usage + 1 + RETURNING usage.usage", + email + ) + .fetch_one(_db) + .await + .map_err(|e| Error::internal_err(format!("updating usage: {e:#}")))?) + } else { + None + }; + Ok((Some(workspace_usage), user_usage)) + }).await.unwrap_or_else(|e| { + tracing::error!("Could not update usage for workspace {workspace_id} and permissioned as {email}, stopped after 10s: {e:#}"); + Err(Error::internal_err(format!("Could not update usage for workspace {workspace_id} and permissioned as {email}, stopped after 10s: {e:#}"))) + }) } else { - (None, None) - }; + Ok((None, None)) + }?; if !premium_workspace { let is_super_admin = diff --git a/backend/windmill-queue/src/jobs_ee.rs b/backend/windmill-queue/src/jobs_oss.rs similarity index 67% rename from backend/windmill-queue/src/jobs_ee.rs rename to backend/windmill-queue/src/jobs_oss.rs index ea1f86b529..987a56bf0d 100644 --- a/backend/windmill-queue/src/jobs_ee.rs +++ b/backend/windmill-queue/src/jobs_oss.rs @@ -1,7 +1,15 @@ +#[cfg(feature = "private")] +#[allow(unused)] +pub use crate::jobs_ee::*; + +#[cfg(not(feature = "private"))] use chrono::{DateTime, Utc}; +#[cfg(not(feature = "private"))] use uuid::Uuid; +#[cfg(not(feature = "private"))] use windmill_common::DB; +#[cfg(not(feature = "private"))] #[allow(dead_code)] pub(crate) async fn update_concurrency_counter( _db: &DB, diff --git a/backend/windmill-queue/src/lib.rs b/backend/windmill-queue/src/lib.rs index 2496bdf818..9b6025e515 100644 --- a/backend/windmill-queue/src/lib.rs +++ b/backend/windmill-queue/src/lib.rs @@ -7,7 +7,9 @@ */ mod jobs; +#[cfg(feature = "private")] pub mod jobs_ee; +pub mod jobs_oss; pub mod schedule; pub use jobs::*; pub mod flow_status; diff --git a/backend/windmill-queue/src/schedule.rs b/backend/windmill-queue/src/schedule.rs index bb7d5df906..294ece7509 100644 --- a/backend/windmill-queue/src/schedule.rs +++ b/backend/windmill-queue/src/schedule.rs @@ -13,7 +13,7 @@ use sqlx::{PgExecutor, Postgres, Transaction}; use std::collections::HashMap; use std::str::FromStr; use windmill_common::db::Authed; -use windmill_common::ee::LICENSE_KEY_VALID; +use windmill_common::ee_oss::LICENSE_KEY_VALID; use windmill_common::flows::Retry; use windmill_common::get_latest_flow_version_info_for_path; use windmill_common::jobs::JobPayload; diff --git a/backend/windmill-worker/Cargo.toml b/backend/windmill-worker/Cargo.toml index dbc8f2ab27..a3b36a5c04 100644 --- a/backend/windmill-worker/Cargo.toml +++ b/backend/windmill-worker/Cargo.toml @@ -10,6 +10,7 @@ path = "src/lib.rs" [features] default = [] +private = [] prometheus = ["dep:prometheus", "windmill-common/prometheus"] enterprise = ["windmill-queue/enterprise", "windmill-git-sync/enterprise", "windmill-common/enterprise", "dep:pem", "dep:tokio-util"] mssql = ["dep:tiberius"] @@ -31,7 +32,7 @@ csharp = ["dep:windmill-parser-csharp"] rust = ["dep:windmill-parser-rust"] nu = ["dep:windmill-parser-nu"] java = ["dep:windmill-parser-java"] -duckdb = [] +duckdb = ["dep:duckdb"] [dependencies] windmill-queue.workspace = true @@ -93,7 +94,7 @@ deno_permissions = { workspace = true, optional = true } deno_io = { workspace = true, optional = true } deno_error = { workspace = true, optional = true } async-stream.workspace = true -duckdb.workspace = true +duckdb = { workspace = true, optional = true } postgres-native-tls.workspace = true native-tls.workspace = true diff --git a/backend/windmill-worker/nsjail/download.rust.config.proto b/backend/windmill-worker/nsjail/download.rust.config.proto new file mode 100644 index 0000000000..7239a77b74 --- /dev/null +++ b/backend/windmill-worker/nsjail/download.rust.config.proto @@ -0,0 +1,117 @@ +name: "rust download script" + +mode: ONCE +hostname: "rust" +log_level: ERROR + +disable_rl: true + +cwd: "/tmp" + +clone_newnet: false + +keep_caps: false +keep_env: true +mount_proc: true + +mount { + src: "/bin" + dst: "/bin" + is_bind: true +} + +mount { + src: "/lib" + dst: "/lib" + is_bind: true +} + + +mount { + src: "/lib64" + dst: "/lib64" + is_bind: true + mandatory: false +} + + +mount { + src: "/usr" + dst: "/usr" + is_bind: true +} + +mount { + src: "/dev/null" + dst: "/dev/null" + is_bind: true + rw: true +} + +mount { + dst: "/tmp" + fstype: "tmpfs" + rw: true + options: "size=500000000" +} + +mount { + src: "/etc" + dst: "/etc" + is_bind: true +} + +mount { + src: "/dev/random" + dst: "/dev/random" + is_bind: true +} +mount { + src: "{JOB_DIR}" + dst: "/tmp" + is_bind: true + rw: true +} + +mount { + src: "/dev/urandom" + dst: "/dev/urandom" + is_bind: true +} + +iface_no_lo: true + +mount { + src: "{BUILD}" + dst: "{BUILD}" + is_bind: true + mandatory: false + rw: true +} +mount { + src: "{CARGO_HOME}" + dst: "{CARGO_HOME}" + is_bind: true + # Readonly + rw: false +} + +mount { + src: "{BUILD}/registry" + dst: "{CARGO_HOME}/registry" + is_bind: true + mandatory: true + # Read-write + rw: true +} + +mount { + src: "{BUILD}/git" + dst: "{CARGO_HOME}/git" + is_bind: true + mandatory: true + # Read-write + rw: true +} + +{DEV} diff --git a/backend/windmill-worker/nsjail/run.rust.config.proto b/backend/windmill-worker/nsjail/run.rust.config.proto index 3357cd88a9..d5265f8ddd 100644 --- a/backend/windmill-worker/nsjail/run.rust.config.proto +++ b/backend/windmill-worker/nsjail/run.rust.config.proto @@ -104,3 +104,6 @@ mount { } {SHARED_MOUNT} + +{DEV} + diff --git a/backend/windmill-worker/src/agent_workers.rs b/backend/windmill-worker/src/agent_workers.rs index 192ae7ae37..f5f5a3b0cc 100644 --- a/backend/windmill-worker/src/agent_workers.rs +++ b/backend/windmill-worker/src/agent_workers.rs @@ -1,3 +1,4 @@ +use reqwest::header::HeaderMap; use uuid::Uuid; use windmill_common::{agent_workers::QueueInitJob, worker::HttpClient}; use windmill_queue::{JobAndPerms, JobCompleted}; @@ -6,14 +7,21 @@ pub async fn queue_init_job(client: &HttpClient, content: &str) -> anyhow::Resul client .post( "/api/agent_workers/queue_init_job", + None, &QueueInitJob { content: content.to_string() }, ) .await .and_then(|x: String| Uuid::parse_str(&x).map_err(|e| anyhow::anyhow!(e))) } -pub async fn pull_job(client: &HttpClient) -> anyhow::Result> { - client.post("/api/agent_workers/pull_job", &()).await +pub async fn pull_job( + client: &HttpClient, + headers: Option, + body: Option, +) -> anyhow::Result> { + client + .post("/api/agent_workers/pull_job", headers, &body) + .await } pub async fn send_result(client: &HttpClient, jc: JobCompleted) -> anyhow::Result { @@ -23,6 +31,7 @@ pub async fn send_result(client: &HttpClient, jc: JobCompleted) -> anyhow::Resul "/api/w/{}/agent_workers/send_result/{}", jc.job.workspace_id, jc.job.id ), + None, &jc, ) .await diff --git a/backend/windmill-worker/src/common.rs b/backend/windmill-worker/src/common.rs index 9955256d13..b419626845 100644 --- a/backend/windmill-worker/src/common.rs +++ b/backend/windmill-worker/src/common.rs @@ -492,6 +492,7 @@ pub async fn update_worker_ping_for_failed_init_script( if let Err(e) = client .post::<_, ()>( UPDATE_PING_URL, + None, &Ping { last_job_executed: Some(last_job_id), last_job_workspace_id: None, @@ -728,7 +729,7 @@ async fn get_workspace_s3_resource_path( storage: Option<&String>, ) -> windmill_common::error::Result> { use windmill_common::{ - job_s3_helpers_ee::get_s3_resource_internal, s3_helpers::StorageResourceType, + job_s3_helpers_oss::get_s3_resource_internal, s3_helpers::StorageResourceType, }; let raw_lfs_opt = if let Some(storage) = storage { @@ -786,7 +787,7 @@ async fn get_workspace_s3_resource_path( get_s3_resource_internal( rt, s3_resource_value_raw, - windmill_common::job_s3_helpers_ee::TokenGenerator::AsClient(client), + windmill_common::job_s3_helpers_oss::TokenGenerator::AsClient(client), db, ) .await @@ -1209,8 +1210,8 @@ pub async fn par_install_language_dependencies<'a>( } #[cfg(all(feature = "enterprise", feature = "parquet"))] let is_not_pro = !matches!( - windmill_common::ee::get_license_plan().await, - windmill_common::ee::LicensePlan::Pro + windmill_common::ee_oss::get_license_plan().await, + windmill_common::ee_oss::LicensePlan::Pro ); #[cfg(all(feature = "enterprise", feature = "parquet"))] if is_not_pro && matches!(install_fn, InstallStrategy::AllAtOnce(_)) { diff --git a/backend/windmill-worker/src/handle_child.rs b/backend/windmill-worker/src/handle_child.rs index 3eb9e95dc9..467fac7c60 100644 --- a/backend/windmill-worker/src/handle_child.rs +++ b/backend/windmill-worker/src/handle_child.rs @@ -53,7 +53,7 @@ use futures::{ use crate::common::{resolve_job_timeout, OccupancyMetrics}; use crate::job_logger::{append_job_logs, append_with_limit}; -use crate::job_logger_ee::process_streaming_log_lines; +use crate::job_logger_oss::process_streaming_log_lines; use crate::worker_utils::{ping_job_status, update_worker_ping_from_job}; use crate::{MAX_RESULT_SIZE, MAX_WAIT_FOR_SIGINT, MAX_WAIT_FOR_SIGTERM}; @@ -227,6 +227,7 @@ pub async fn handle_child( if let Err(err) = client .post::<_, ()>( &format!("/api/agent_workers/set_job_cancelled/{}", job_id), + None, &JobCancelled { canceled_by: "timeout".to_string(), reason: format!("duration > {}", timeout_duration.as_secs()), diff --git a/backend/windmill-worker/src/job_logger.rs b/backend/windmill-worker/src/job_logger.rs index f4abd1d0c2..626dfbce44 100644 --- a/backend/windmill-worker/src/job_logger.rs +++ b/backend/windmill-worker/src/job_logger.rs @@ -13,10 +13,10 @@ use std::sync::Arc; use uuid::Uuid; #[cfg(not(all(feature = "enterprise", feature = "parquet")))] -use crate::job_logger_ee::default_disk_log_storage; +use crate::job_logger_oss::default_disk_log_storage; #[cfg(all(feature = "enterprise", feature = "parquet"))] -use crate::job_logger_ee::s3_storage; +use crate::job_logger_oss::s3_storage; pub enum CompactLogs { #[cfg(not(all(feature = "enterprise", feature = "parquet")))] diff --git a/backend/windmill-worker/src/job_logger_ee.rs b/backend/windmill-worker/src/job_logger_oss.rs similarity index 68% rename from backend/windmill-worker/src/job_logger_ee.rs rename to backend/windmill-worker/src/job_logger_oss.rs index 22772878ee..3ee3d29a3a 100644 --- a/backend/windmill-worker/src/job_logger_ee.rs +++ b/backend/windmill-worker/src/job_logger_oss.rs @@ -1,13 +1,14 @@ -use std::io; -use std::sync::atomic::AtomicU32; -use std::sync::Arc; +#[cfg(feature = "private")] +#[allow(unused)] +pub use crate::job_logger_ee::*; -use uuid::Uuid; -use windmill_common::DB; +#[cfg(not(feature = "private"))] +use { + crate::job_logger::CompactLogs, std::io, std::sync::atomic::AtomicU32, std::sync::Arc, + uuid::Uuid, windmill_common::DB, +}; -use crate::job_logger::CompactLogs; - -#[cfg(all(feature = "enterprise", feature = "parquet"))] +#[cfg(all(feature = "enterprise", feature = "parquet", not(feature = "private")))] pub(crate) async fn s3_storage( _job_id: &Uuid, _w_id: &str, @@ -19,6 +20,7 @@ pub(crate) async fn s3_storage( tracing::info!("Logs length of {_job_id} has exceeded a threshold. Implementation to store excess on s3 in not OSS"); } +#[cfg(not(feature = "private"))] #[allow(dead_code)] pub(crate) async fn default_disk_log_storage( job_id: &Uuid, @@ -32,6 +34,7 @@ pub(crate) async fn default_disk_log_storage( tracing::info!("Logs length of {job_id} has exceeded a threshold. Implementation to store excess on disk in not OSS"); } +#[cfg(not(feature = "private"))] pub(crate) fn process_streaming_log_lines( r: Result, io::Error>, _stderr: bool, diff --git a/backend/windmill-worker/src/lib.rs b/backend/windmill-worker/src/lib.rs index fa6eb10960..3dfeba5c42 100644 --- a/backend/windmill-worker/src/lib.rs +++ b/backend/windmill-worker/src/lib.rs @@ -27,7 +27,9 @@ mod go_executor; mod graphql_executor; mod handle_child; pub mod job_logger; -mod job_logger_ee; +#[cfg(feature = "private")] +pub mod job_logger_ee; +mod job_logger_oss; mod js_eval; #[cfg(feature = "mysql")] mod mysql_executor; @@ -35,7 +37,9 @@ mod mysql_executor; mod nu_executor; #[cfg(feature = "oracledb")] mod oracledb_executor; -mod otel_ee; +#[cfg(feature = "private")] +pub mod otel_ee; +mod otel_oss; mod pg_executor; #[cfg(feature = "php")] mod php_executor; @@ -53,9 +57,8 @@ mod worker_flow; mod worker_lockfiles; mod worker_utils; -pub use worker_lockfiles::process_relative_imports; - pub use worker::*; +pub use worker_lockfiles::process_relative_imports; pub use result_processor::handle_job_error; diff --git a/backend/windmill-worker/src/otel_ee.rs b/backend/windmill-worker/src/otel_ee.rs deleted file mode 100644 index 311ffd6355..0000000000 --- a/backend/windmill-worker/src/otel_ee.rs +++ /dev/null @@ -1,3 +0,0 @@ -use windmill_queue::MiniPulledJob; - -pub fn add_root_flow_job_to_otlp(_queued_job: &MiniPulledJob, _success: bool) {} diff --git a/backend/windmill-worker/src/otel_oss.rs b/backend/windmill-worker/src/otel_oss.rs new file mode 100644 index 0000000000..2f65c534b4 --- /dev/null +++ b/backend/windmill-worker/src/otel_oss.rs @@ -0,0 +1,9 @@ +#[cfg(feature = "private")] +#[allow(unused)] +pub use crate::otel_ee::*; + +#[cfg(not(feature = "private"))] +use windmill_queue::MiniPulledJob; + +#[cfg(not(feature = "private"))] +pub fn add_root_flow_job_to_otlp(_queued_job: &MiniPulledJob, _success: bool) {} diff --git a/backend/windmill-worker/src/python_executor.rs b/backend/windmill-worker/src/python_executor.rs index 76ee967656..35815e452c 100644 --- a/backend/windmill-worker/src/python_executor.rs +++ b/backend/windmill-worker/src/python_executor.rs @@ -20,7 +20,7 @@ use tokio::{ }; use uuid::Uuid; #[cfg(all(feature = "enterprise", feature = "parquet", unix))] -use windmill_common::ee::{get_license_plan, LicensePlan}; +use windmill_common::ee_oss::{get_license_plan, LicensePlan}; use windmill_common::{ error::{ self, @@ -658,7 +658,7 @@ except BaseException as e: // Add /tmp/windmill/cache/python_x_y_z/global-site-packages to PYTHONPATH. // Usefull if certain wheels needs to be preinstalled before execution. - let global_site_packages_path = py_version.to_cache_dir() + "/global-site-packages"; + let global_site_packages_path = py_version.to_cache_dir(true) + "/global-site-packages"; let additional_python_paths_folders = { let mut paths = additional_python_paths.clone(); if std::fs::metadata(&global_site_packages_path).is_ok() { @@ -1489,7 +1489,7 @@ pub async fn handle_python_reqs( if req.starts_with('#') || req.starts_with('-') || req.trim().is_empty() { continue; } - let py_prefix = &py_version.to_cache_dir(); + let py_prefix = &py_version.to_cache_dir(false); let venv_p = format!( "{py_prefix}/{}", @@ -1740,7 +1740,7 @@ pub async fn handle_python_reqs( tokio::select! { // Cancel was called on the job _ = kill_rx.recv() => return Err(anyhow::anyhow!("S3 pull was canceled")), - pull = pull_from_tar(os, venv_p.clone(), py_version.to_cache_dir_top_level(), None, false) => { + pull = pull_from_tar(os, venv_p.clone(), py_version.to_cache_dir_top_level(false), None, false) => { if let Err(e) = pull { tracing::info!( workspace_id = %w_id, @@ -1891,7 +1891,7 @@ pub async fn handle_python_reqs( #[cfg(all(feature = "enterprise", feature = "parquet", unix))] if s3_push { if let Some(os) = windmill_common::s3_helpers::get_object_store().await { - tokio::spawn(build_tar_and_push(os, venv_p.clone(), py_version.to_cache_dir_top_level(), None, false)); + tokio::spawn(build_tar_and_push(os, venv_p.clone(), py_version.to_cache_dir_top_level(false), None, false)); } } diff --git a/backend/windmill-worker/src/python_versions.rs b/backend/windmill-worker/src/python_versions.rs index 26be7aca65..2f12d34dc6 100644 --- a/backend/windmill-worker/src/python_versions.rs +++ b/backend/windmill-worker/src/python_versions.rs @@ -153,12 +153,18 @@ impl PyV { let valid = all_versions .clone() .into_iter() - .filter(|v| version_specifiers.iter().all(|vs| vs.contains(&*v))) + .filter(|v| version_specifiers.iter().all(|vs| (vs).contains(&*v))) .collect_vec(); if !valid.is_empty() { + let mut result = valid[0].clone(); + // Is there at least one version specifier that has PATCH digit? + let patch_vs = version_specifiers + .iter() + .any(|vs| vs.version().release().get(2).is_some()); + if select_latest { - return Ok(valid[0].clone()); + return Ok(result.clone()); } // Usually INSTANCE_PYTHON_VERSION @@ -183,9 +189,10 @@ impl PyV { // - We will iterate until find the closest version to target. // - If closest version has the same MINOR version, use it. // - If it differs in MINOR version, take latest PATCH version. - // - let mut result = None; + let [major, minor, ..] = result.release() else { + return Err(Error::InternalErr(format!("Failed to parse \"{}\". Available python versions are supposed to be in SEMVER format (MAJOR.MINOR)", *result))); + }; // This represents newest version with oldest MINOR: // // I Iterable Newest in MINOR @@ -195,12 +202,9 @@ impl PyV { // 4. 3.10.2 -> 3.10.2 // 5. 3.10.1 -> 3.10.2 // 6. 3.10.0 -> 3.10.2 - let mut newest_in_minor = None; - for v in valid.iter() { - if result.is_none() { - result.replace(v); - } + let mut newest_in_minor = (result.clone(), (*major, *minor)); + for v in valid.iter() { if v < &gv { // We will not continue if we start looking into versions older than gravity version. break; @@ -212,37 +216,46 @@ impl PyV { // Since we go top to down we can assume // the first occurence of new minor version contains the latest patch version. - if matches!(newest_in_minor, Some((_, mm)) if mm != (major, minor)) - || newest_in_minor.is_none() - { - newest_in_minor = Some((v.clone(), (major, minor))); + if newest_in_minor.1 != (*major, *minor) { + newest_in_minor = (v.clone(), (*major, *minor)); } if gravity_matcher.contains(v) { // return as soon as gravity matcher has first hit. - return Ok(v.clone()); + // Only in case version specifiers do specify PATCH version OR gravity version specify PATCH + if patch_vs || gv.release().get(2).is_some() { + return Ok(v.clone()); + } else { + let Some(release_numbers) = v.release().get(0..=1) else { + return Err(Error::InternalErr(format!( + "Failed to get release numbers from: \"{}\". ", + **v + ))); + }; + return Ok(PyV(pep440_rs::Version::new(release_numbers))); + } } // If we are still in the loop, it means that we are getting closer to gravity version else { - result = Some(v); + result = v.clone(); } } let [gravity_major, gravity_minor, ..] = gv.release() else { - return Err(Error::internal_err(format!("Cannot get MAJOR or/and MINOR version of python gravity version ({}). Something might be wrong with INSTANCE_PYTHON_VERSION.", &*gv))); + return Err(Error::internal_err(format!("Cannot get MAJOR nor MINOR version of python gravity version ({}). Something might be wrong with INSTANCE_PYTHON_VERSION.", &*gv))); }; - if let Some((v, mm)) = newest_in_minor { - if (gravity_major, gravity_minor) != mm { - return Ok(v); + if (*gravity_major, *gravity_minor) != newest_in_minor.1 { + // Return full version only if there is PATCH versions in version specifiers + if patch_vs { + return Ok(newest_in_minor.0); + } else { + let mm = newest_in_minor.1; + return Ok(PyV(pep440_rs::Version::new([mm.0, mm.1]))); } } - result - .ok_or(Error::internal_err( - "No python candidates found. This is a bug!", - )) - .map(ToOwned::to_owned) + Ok(result) } else { Err(anyhow!( " @@ -265,14 +278,24 @@ impl PyV { .into()) } } - /// e.g.: `/tmp/windmill/cache/python_3xy` - pub(crate) fn to_cache_dir(&self) -> String { + /// e.g.: `/tmp/windmill/cache/python_3_x_y` + pub(crate) fn to_cache_dir(&self, ignore_patch: bool) -> String { use windmill_common::worker::ROOT_CACHE_DIR; - format!("{ROOT_CACHE_DIR}{}", self.to_cache_dir_top_level()) + format!( + "{ROOT_CACHE_DIR}{}", + self.to_cache_dir_top_level(ignore_patch) + ) } /// e.g.: `python_3_x_y` - pub fn to_cache_dir_top_level(&self) -> String { + pub fn to_cache_dir_top_level(&self, ignore_patch: bool) -> String { + if ignore_patch { + if let [major, minor, ..] = self.release() { + return format!("python_{major}_{minor}"); + } + + tracing::warn!("failed to parse python's ({}) top level directory with no patch digit, fallback to full version.", self.to_string()); + } format!("python_{}", self.to_string().replace(".", "_")) } @@ -283,6 +306,7 @@ impl PyV { ) -> Self { let mut err = None; let pyv = match INSTANCE_PYTHON_VERSION.read().await.clone() { + Some(v) if &v == "default" => PyVAlias::default().into(), Some(v) => pep440_rs::Version::from_str(&v).unwrap_or_else(|_| { let v = PyVAlias::default().into(); err = Some(format!("\nCannot parse INSTANCE_PYTHON_VERSION ({:?}), fallback to latest_stable ({v:?})", *INSTANCE_PYTHON_VERSION)); @@ -575,7 +599,7 @@ impl PyV { // For the default version directory created during startup (main.rs) DirBuilder::new() .recursive(true) - .create(self.to_cache_dir()) + .create(self.to_cache_dir(false)) .await .expect("could not create initial worker dir"); @@ -762,7 +786,7 @@ mod tests { pyv("0.9.3"), pyv("0.9.2"), ], - pyv("0.9.4"), // + pyv("0.9"), // ) .await; } @@ -785,7 +809,7 @@ mod tests { pyv("0.8.1"), pyv("0.8.0"), ], - pyv("1.0.2"), // + pyv("1.0"), // ) .await; } @@ -824,7 +848,7 @@ mod tests { pyv("2.2.1"), pyv("2.2.0"), ], - pyv("2.2.2"), + pyv("2.2"), ) .await; } @@ -845,4 +869,151 @@ mod tests { ) .await; } + #[tokio::test] + async fn test_python_resolution_8() { + assert_resolution( + "2.2", + false, + vec![">2.2", ">=2.4", "<2.4.1"], + vec![ + pyv("2.4.1"), + pyv("2.4.0"), + pyv("2.3.1"), + pyv("2.3.0"), + pyv("2.2.1"), + pyv("2.2.0"), + pyv("2.1.1"), + pyv("2.1.0"), + ], + pyv("2.4.0"), + ) + .await; + } + #[tokio::test] + async fn test_python_resolution_9() { + assert_resolution( + "2.2", + true, + vec![">2.2", ">2.3", "<2.4.1"], + vec![ + pyv("2.4.1"), + pyv("2.4.0"), + pyv("2.3.1"), + pyv("2.3.0"), + pyv("2.2.1"), + pyv("2.2.0"), + pyv("2.1.1"), + pyv("2.1.0"), + ], + pyv("2.4.0"), + ) + .await; + } + #[tokio::test] + async fn test_python_resolution_10() { + assert_resolution( + "2.2", + false, + vec![">2.2", ">=2.4"], + vec![ + pyv("2.4.1"), + pyv("2.4.0"), + pyv("2.3.1"), + pyv("2.3.0"), + pyv("2.2.1"), + pyv("2.2.0"), + pyv("2.1.1"), + pyv("2.1.0"), + ], + // vec![pyv("2.4.1"), pyv("2.3"), pyv("2.2"), pyv("2.1")], + pyv("2.4"), + ) + .await; + } + #[tokio::test] + async fn test_python_resolution_11() { + assert_resolution( + "2.4.1", + false, + vec![">2.2", ">=2.3", "<2.4"], + vec![ + pyv("2.4.1"), + pyv("2.4.0"), + pyv("2.3.1"), + pyv("2.3.0"), + pyv("2.2.1"), + pyv("2.2.0"), + pyv("2.1.1"), + pyv("2.1.0"), + ], + // vec![pyv("2.4.1"), pyv("2.3"), pyv("2.2"), pyv("2.1")], + pyv("2.3"), + ) + .await; + } + #[tokio::test] + async fn test_python_resolution_12() { + assert_resolution( + "2.3", + false, + vec![">2.2", ">=2.3", "<2.4"], + vec![ + pyv("2.4.1"), + pyv("2.4.0"), + pyv("2.3.1"), + pyv("2.3.0"), + pyv("2.2.1"), + pyv("2.2.0"), + pyv("2.1.1"), + pyv("2.1.0"), + ], + // vec![pyv("2.4.1"), pyv("2.3"), pyv("2.2"), pyv("2.1")], + pyv("2.3"), + ) + .await; + } + #[tokio::test] + async fn test_python_resolution_13() { + assert_resolution( + "2.3.1", + false, + vec![">2.2", ">=2.3", "<2.4"], + vec![ + pyv("2.4.1"), + pyv("2.4.0"), + pyv("2.3.1"), + pyv("2.3.0"), + pyv("2.2.1"), + pyv("2.2.0"), + pyv("2.1.1"), + pyv("2.1.0"), + ], + // vec![pyv("2.4.1"), pyv("2.3"), pyv("2.2"), pyv("2.1")], + pyv("2.3.1"), + ) + .await; + } + #[tokio::test] + async fn test_python_resolution_14() { + assert_resolution( + "2.3.0", + false, + vec![], + vec![pyv("2.4.1"), pyv("2.4.0"), pyv("2.3.1")], + pyv("2.3.1"), + ) + .await; + } + + #[tokio::test] + async fn test_python_resolution_16() { + assert_resolution( + "2.3", + false, + vec![], + vec![pyv("2.4.1"), pyv("2.4.0"), pyv("2.3.1")], + pyv("2.3"), + ) + .await; + } } diff --git a/backend/windmill-worker/src/result_processor.rs b/backend/windmill-worker/src/result_processor.rs index 342c53a301..5056851389 100644 --- a/backend/windmill-worker/src/result_processor.rs +++ b/backend/windmill-worker/src/result_processor.rs @@ -12,7 +12,7 @@ use std::{ }; use tracing::{field, Instrument}; #[cfg(not(feature = "otel"))] -use windmill_common::otel_ee::FutureExt; +use windmill_common::otel_oss::FutureExt; use uuid::Uuid; @@ -32,7 +32,7 @@ use windmill_queue::{ append_logs, get_queued_job, CanceledBy, JobCompleted, MiniPulledJob, WrappedError, }; -use serde_json::{json, value::RawValue}; +use serde_json::{json, value::RawValue, Value}; use tokio::task::JoinHandle; @@ -41,10 +41,10 @@ use windmill_queue::{add_completed_job, add_completed_job_error}; use crate::{ bash_executor::ANSI_ESCAPE_RE, common::{error_to_value, read_result, save_in_cache}, - otel_ee::add_root_flow_job_to_otlp, + otel_oss::add_root_flow_job_to_otlp, worker_flow::update_flow_status_after_job_completion, JobCompletedReceiver, JobCompletedSender, SameWorkerSender, SendResult, UpdateFlow, - INIT_SCRIPT_TAG, + INIT_SCRIPT_TAG, SAME_WORKER_REQUIREMENTS, }; use windmill_common::client::AuthedClient; @@ -54,7 +54,7 @@ async fn process_jc( base_internal_url: &str, db: &DB, worker_dir: &str, - same_worker_tx: &SameWorkerSender, + same_worker_tx: Option<&SameWorkerSender>, job_completed_sender: &JobCompletedSender, #[cfg(feature = "benchmark")] bench: &mut BenchmarkIter, ) { @@ -76,7 +76,7 @@ async fn process_jc( } else { jc.job.id }; - windmill_common::otel_ee::set_span_parent(&span, &rj); + windmill_common::otel_oss::set_span_parent(&span, &rj); if let Some(lg) = jc.job.script_lang.as_ref() { span.record("language", lg.as_str()); @@ -104,8 +104,8 @@ async fn process_jc( jc, &base_internal_url, &db, - &worker_dir, - &same_worker_tx, + worker_dir, + same_worker_tx, &worker_name, job_completed_sender.clone(), #[cfg(feature = "benchmark")] @@ -119,6 +119,11 @@ async fn process_jc( } } +enum JobCompletedRx { + JobCompleted(SendResult), + Killpill, +} + pub fn start_background_processor( job_completed_rx: JobCompletedReceiver, job_completed_sender: JobCompletedSender, @@ -140,10 +145,6 @@ pub fn start_background_processor( #[cfg(feature = "benchmark")] let mut infos = BenchmarkInfo::new(); - enum JobCompletedRx { - JobCompleted(SendResult), - Killpill, - } //if we have been killed, we want to drain the queue of jobs while let Some(sr) = { if has_been_killed && same_worker_queue_size.load(Ordering::SeqCst) == 0 { @@ -186,7 +187,7 @@ pub fn start_background_processor( &base_internal_url, &db, &worker_dir, - &same_worker_tx, + Some(&same_worker_tx), &job_completed_sender, #[cfg(feature = "benchmark")] &mut bench, @@ -229,19 +230,19 @@ pub fn start_background_processor( tracing::info!(parent_flow = %flow, "updating flow status"); if let Err(e) = update_flow_status_after_job_completion( &db, - &AuthedClient { - base_internal_url: base_internal_url.to_string(), - workspace: w_id.clone(), - token: token.clone(), - force_client: None, - }, + &AuthedClient::new( + base_internal_url.to_string(), + w_id.clone(), + token.clone(), + None, + ), flow, &Uuid::nil(), &w_id, success, Arc::new(result), true, - same_worker_tx.clone(), + &same_worker_tx, &worker_dir, stop_early_override, &worker_name, @@ -276,7 +277,7 @@ pub fn start_background_processor( async fn send_job_completed(job_completed_tx: JobCompletedSender, jc: JobCompleted) { job_completed_tx .send_job(jc, true) - .with_context(windmill_common::otel_ee::otel_ctx()) + .with_context(windmill_common::otel_oss::otel_ctx()) .await .expect("send job completed") } @@ -312,7 +313,7 @@ pub async fn process_result( duration, }, ) - .with_context(windmill_common::otel_ee::otel_ctx()) + .with_context(windmill_common::otel_oss::otel_ctx()) .await; Ok(true) } @@ -374,7 +375,7 @@ pub async fn process_result( duration, }, ) - .with_context(windmill_common::otel_ee::otel_ctx()) + .with_context(windmill_common::otel_oss::otel_ctx()) .await; Ok(false) } @@ -386,23 +387,19 @@ pub async fn handle_receive_completed_job( base_internal_url: &str, db: &DB, worker_dir: &str, - same_worker_tx: &SameWorkerSender, + same_worker_tx: Option<&SameWorkerSender>, worker_name: &str, job_completed_tx: JobCompletedSender, #[cfg(feature = "benchmark")] bench: &mut BenchmarkIter, ) -> Option> { let token = jc.token.clone(); let workspace = jc.job.workspace_id.clone(); - let client = AuthedClient { - base_internal_url: base_internal_url.to_string(), - workspace, - token, - force_client: None, - }; + let client = AuthedClient::new(base_internal_url.to_string(), workspace, token, None); let job = jc.job.clone(); let mem_peak = jc.mem_peak.clone(); let canceled_by = jc.canceled_by.clone(); - match process_completed_job( + + let processed_completed_job = process_completed_job( jc, &client, db, @@ -413,8 +410,9 @@ pub async fn handle_receive_completed_job( #[cfg(feature = "benchmark")] bench, ) - .await - { + .await; + + match processed_completed_job { Err(err) => { handle_job_error( db, @@ -454,7 +452,7 @@ pub async fn process_completed_job( client: &AuthedClient, db: &DB, worker_dir: &str, - same_worker_tx: SameWorkerSender, + same_worker_tx: Option<&SameWorkerSender>, worker_name: &str, job_completed_tx: JobCompletedSender, #[cfg(feature = "benchmark")] bench: &mut BenchmarkIter, @@ -530,7 +528,7 @@ pub async fn process_completed_job( true, result, false, - same_worker_tx.clone(), + &same_worker_tx.expect(SAME_WORKER_REQUIREMENTS).to_owned(), &worker_dir, None, worker_name, @@ -570,7 +568,7 @@ pub async fn process_completed_job( false, Arc::new(serde_json::value::to_raw_value(&result).unwrap()), false, - same_worker_tx, + &same_worker_tx.expect(SAME_WORKER_REQUIREMENTS).to_owned(), &worker_dir, None, worker_name, @@ -587,6 +585,34 @@ pub async fn process_completed_job( return Ok(None); } +async fn handle_non_flow_job_error( + db: &DB, + job: &MiniPulledJob, + mem_peak: i32, + canceled_by: Option, + err: Value, + worker_name: &str, +) -> Result { + append_logs( + &job.id, + &job.workspace_id, + format!("Unexpected error during job execution:\n{err:#?}"), + &db.into(), + ) + .await; + add_completed_job_error( + db, + job, + mem_peak, + canceled_by, + err, + worker_name, + false, + None, + ) + .await +} + #[tracing::instrument(name = "job_error", level = "info", skip_all, fields(job_id = %job.id))] pub async fn handle_job_error( db: &DB, @@ -596,7 +622,7 @@ pub async fn handle_job_error( canceled_by: Option, err: Error, unrecoverable: bool, - same_worker_tx: SameWorkerSender, + same_worker_tx: Option<&SameWorkerSender>, worker_dir: &str, worker_name: &str, job_completed_tx: JobCompletedSender, @@ -605,22 +631,13 @@ pub async fn handle_job_error( let err = error_to_value(err); let update_job_future = || async { - append_logs( - &job.id, - &job.workspace_id, - format!("Unexpected error during job execution:\n{err:#?}"), - &db.into(), - ) - .await; - add_completed_job_error( + handle_non_flow_job_error( db, job, mem_peak, canceled_by.clone(), err.clone(), worker_name, - false, - None, ) .await }; @@ -649,7 +666,7 @@ pub async fn handle_job_error( false, Arc::new(serde_json::value::to_raw_value(&wrapped_error).unwrap()), unrecoverable, - same_worker_tx, + &same_worker_tx.expect(SAME_WORKER_REQUIREMENTS).clone(), worker_dir, None, worker_name, diff --git a/backend/windmill-worker/src/rust_executor.rs b/backend/windmill-worker/src/rust_executor.rs index 1a6a93ecca..045158511e 100644 --- a/backend/windmill-worker/src/rust_executor.rs +++ b/backend/windmill-worker/src/rust_executor.rs @@ -4,7 +4,11 @@ use uuid::Uuid; use windmill_parser_rust::parse_rust_deps_into_manifest; use itertools::Itertools; -use tokio::{fs::File, io::AsyncReadExt, process::Command}; +use tokio::{ + fs::{create_dir_all, File}, + io::AsyncReadExt, + process::Command, +}; use windmill_common::{ error::{self, Error}, utils::calculate_hash, @@ -19,8 +23,8 @@ use crate::{ read_result, start_child_process, OccupancyMetrics, }, handle_child::handle_child, - DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV, NSJAIL_PATH, PATH_ENV, PROXY_ENVS, - RUST_CACHE_DIR, TZ_ENV, + DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV, NSJAIL_PATH, PATH_ENV, PROXY_ENVS, RUST_CACHE_DIR, + TZ_ENV, }; use windmill_common::client::AuthedClient; @@ -28,12 +32,18 @@ use windmill_common::client::AuthedClient; use crate::SYSTEM_ROOT; const NSJAIL_CONFIG_RUN_RUST_CONTENT: &str = include_str!("../nsjail/run.rust.config.proto"); +const NSJAIL_CONFIG_COMPILE_RUST_CONTENT: &str = + include_str!("../nsjail/download.rust.config.proto"); lazy_static::lazy_static! { static ref HOME_DIR: String = std::env::var("HOME").expect("Could not find the HOME environment variable"); static ref CARGO_HOME: String = std::env::var("CARGO_HOME").unwrap_or_else(|_| { CARGO_HOME_DEFAULT.clone() }); static ref RUSTUP_HOME: String = std::env::var("RUSTUP_HOME").unwrap_or_else(|_| { RUSTUP_HOME_DEFAULT.clone() }); static ref CARGO_PATH: String = std::env::var("CARGO_PATH").unwrap_or_else(|_| format!("{}/bin/cargo", CARGO_HOME.as_str())); + // static ref CARGO_SWEEP_PATH: String = std::env::var("CARGO_SWEEP_PATH").unwrap_or_else(|_| format!("{}/bin/cargo-sweep", CARGO_HOME.as_str())); + static ref SWEEP_MAXSIZE: String = std::env::var("CARGO_SWEEP_MAXSIZE").unwrap_or("25GB".to_owned()); + static ref NO_SHARED_BUILD_DIR: bool = std::env::var("RUST_NO_SHARED_BUILD_DIR").ok().map(|flag| flag == "true").unwrap_or(false); + } #[cfg(windows)] @@ -42,7 +52,21 @@ lazy_static::lazy_static! { static ref RUSTUP_HOME_DEFAULT: String = format!("{}\\.rustup", *HOME_DIR); } -#[cfg(unix)] +#[cfg(debug_assertions)] +const DEV_CONF_NSJAIL: &'static str = r#" +# Mount nix store for nixos to work properly +mount { + src: "/nix/store" + dst: "/nix/store" + is_bind: true + mandatory: false +} +"#; + +#[cfg(not(debug_assertions))] +const DEV_CONF_NSJAIL: &'static str = ""; + +#[cfg(not(windows))] lazy_static::lazy_static! { static ref CARGO_HOME_DEFAULT: String = format!("{}/.cargo", *HOME_DIR); static ref RUSTUP_HOME_DEFAULT: String = format!("{}/.rustup", *HOME_DIR); @@ -176,54 +200,211 @@ pub async fn generate_cargo_lockfile( Ok(req_content) } +async fn get_build_dir( + job: &MiniPulledJob, + job_dir: &str, + conn: &Connection, + worker_name: &str, + is_preview: bool, +) -> anyhow::Result { + let (bd, run_sweep) = job + .runnable_path + .as_ref() + .and_then(|p| { + if !is_preview || *NO_SHARED_BUILD_DIR { + None + } else { + if *DISABLE_NSJAIL { + // If nsjail is disabled then entire worker has shared build directory + // It drastically improves cache hit-rate. + Some((format!("{RUST_CACHE_DIR}/build/{worker_name}"), true)) + } else { + // If nsjail is enabled, having global shared directory is vulnerability and target for an attack + // Instead we either: + // 1. Create different build directory for workspace script and user. Balanced caching while mainining high degree of security. + // 2. If user is not known or something else goes wrong - use random build dir. This is equivalent to no cache at all. + Some(( + format!( + "{RUST_CACHE_DIR}/build/{}@{}@{}", + &job.workspace_id, + p.replace('/', "."), + &job.created_by + ), + true, + )) + } + } + }) + .unwrap_or((format!("{RUST_CACHE_DIR}/build/{}", Uuid::new_v4()), false)); + + { + let (t, r, g) = ( + create_dir_all(format!("{}/target", &bd)).await, + create_dir_all(format!("{}/registry", &bd)).await, + create_dir_all(format!("{}/git", &bd)).await, + ); + + t.and(r) + .and(g) + .map_err(|e| anyhow::anyhow!("Could not create build dir for rust.\ne: {e}"))?; + } + + if run_sweep { + // Also run sweep to make sure target isn't using too much disk + let mut sweep_cmd = Command::new(CARGO_PATH.as_str()); + sweep_cmd + .current_dir(job_dir) + .env_clear() + .env("PATH", PATH_ENV.as_str()) + .env("CARGO_HOME", CARGO_HOME.as_str()) + .env("HOME", HOME_ENV.as_str()) + .env("CARGO_TARGET_DIR", &(bd.clone() + "/target")) + .env("RUSTUP_HOME", RUSTUP_HOME.as_str()) + .args(["sweep", "--maxsize", SWEEP_MAXSIZE.as_str()]) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + + #[cfg(windows)] + { + sweep_cmd.env("SystemRoot", SYSTEM_ROOT.as_str()); + sweep_cmd.env( + "TMP", + std::env::var("TMP").unwrap_or_else(|_| "C:\\tmp".to_string()), + ); + sweep_cmd.env("USERPROFILE", crate::USERPROFILE_ENV.as_str()); + } + + let (job_id, conn, w_id, wk_name) = ( + job.id.clone(), + conn.clone(), + job.workspace_id.clone(), + worker_name.to_owned(), + ); + + tokio::spawn(async move { + if let Err(e) = match start_child_process(sweep_cmd, CARGO_PATH.as_str()).await { + Ok(sweep_process) => { + handle_child( + &job_id, + &conn, + &mut 0, + &mut None, + sweep_process, + false, + &wk_name, + &w_id, + "cargo sweep", + None, + false, + &mut None, + None, + ) + .await + } + Err(e) => Err(e), + } { + tracing::warn!( + workspace_id = %w_id, + job_id = %job_id, + "Failed to run `cargo sweep`. Rust cache may grow over time, cargo sweep is meant to clean up unused cache.\ne: {e}\n" + ); + } + }); + } + Ok(bd) +} + pub async fn build_rust_crate( - job_id: &Uuid, + job: &MiniPulledJob, mem_peak: &mut i32, canceled_by: &mut Option, job_dir: &str, conn: &Connection, worker_name: &str, - w_id: &str, base_internal_url: &str, hash: &str, occupancy_metrics: &mut OccupancyMetrics, + is_preview: bool, ) -> error::Result { let bin_path = format!("{}/{hash}", RUST_CACHE_DIR); - let mut build_rust_cmd = Command::new(CARGO_PATH.as_str()); - build_rust_cmd - .current_dir(job_dir) - .env_clear() - .envs(PROXY_ENVS.clone()) - .env("PATH", PATH_ENV.as_str()) - .env("BASE_INTERNAL_URL", base_internal_url) - .env("HOME", HOME_ENV.as_str()) - .env("CARGO_HOME", CARGO_HOME.as_str()) - .env("RUSTUP_HOME", RUSTUP_HOME.as_str()) - .args(vec!["build", "--release"]) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()); + let build_dir = get_build_dir(job, job_dir, conn, worker_name, is_preview).await?; - #[cfg(windows)] - { - build_rust_cmd.env("SystemRoot", SYSTEM_ROOT.as_str()); - build_rust_cmd.env( - "TMP", - std::env::var("TMP").unwrap_or_else(|_| "C:\\tmp".to_string()), - ); - build_rust_cmd.env("USERPROFILE", crate::USERPROFILE_ENV.as_str()); - } + let child = if !*DISABLE_NSJAIL { + let _ = write_file( + job_dir, + "download.config.proto", + &NSJAIL_CONFIG_COMPILE_RUST_CONTENT + .replace("{JOB_DIR}", job_dir) + .replace("{CACHE_DIR}", RUST_CACHE_DIR) + .replace("{CARGO_HOME}", CARGO_HOME.as_str()) + .replace("{DEV}", DEV_CONF_NSJAIL) + .replace("{BUILD}", &build_dir), + )?; + let mut nsjail_cmd = Command::new(NSJAIL_PATH.as_str()); + nsjail_cmd + .current_dir(job_dir) + .env_clear() + .env("PATH", PATH_ENV.as_str()) + .env("TZ", TZ_ENV.as_str()) + .env("BASE_INTERNAL_URL", base_internal_url) + .envs(PROXY_ENVS.clone()) + .env("HOME", HOME_ENV.as_str()) + .env("CARGO_HOME", CARGO_HOME.as_str()) + .env("RUSTUP_HOME", RUSTUP_HOME.as_str()) + .env("CARGO_TARGET_DIR", &(build_dir.clone() + "/target")) + .args(vec![ + "--config", + "download.config.proto", + "--", + CARGO_PATH.as_ref(), + "build", + ]) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + if !is_preview { + nsjail_cmd.arg("--release"); + } + start_child_process(nsjail_cmd, NSJAIL_PATH.as_str()).await? + } else { + let mut build_rust_cmd = Command::new(CARGO_PATH.as_str()); + build_rust_cmd + .current_dir(job_dir) + .env_clear() + .envs(PROXY_ENVS.clone()) + .env("PATH", PATH_ENV.as_str()) + .env("BASE_INTERNAL_URL", base_internal_url) + .env("HOME", HOME_ENV.as_str()) + .env("CARGO_HOME", CARGO_HOME.as_str()) + .env("RUSTUP_HOME", RUSTUP_HOME.as_str()) + .env("CARGO_TARGET_DIR", &(build_dir.clone() + "/target")) + .args(vec!["build"]) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); - let build_rust_process = start_child_process(build_rust_cmd, CARGO_PATH.as_str()).await?; + if !is_preview { + build_rust_cmd.arg("--release"); + } + #[cfg(windows)] + { + build_rust_cmd.env("SystemRoot", SYSTEM_ROOT.as_str()); + build_rust_cmd.env( + "TMP", + std::env::var("TMP").unwrap_or_else(|_| "C:\\tmp".to_string()), + ); + build_rust_cmd.env("USERPROFILE", crate::USERPROFILE_ENV.as_str()); + } + start_child_process(build_rust_cmd, CARGO_PATH.as_str()).await? + }; handle_child( - job_id, + &job.id, conn, mem_peak, canceled_by, - build_rust_process, + child, false, worker_name, - w_id, + &job.workspace_id, "rust build", None, false, @@ -231,16 +412,19 @@ pub async fn build_rust_crate( None, ) .await?; - append_logs(job_id, w_id, "\n\n", conn).await; + append_logs(&job.id, &job.workspace_id, "\n\n", conn).await; tokio::fs::copy( - &format!("{job_dir}/target/release/main"), + &format!( + "{build_dir}/target/{}/main", + if is_preview { "debug" } else { "release" }, + ), format! {"{job_dir}/main"}, ) .await .map_err(|e| { Error::ExecutionErr(format!( - "could not copy built binary from [...]/target/release/main to {job_dir}/main: {e:?}" + "could not copy built binary from [...]/target/.../main to {job_dir}/main: {e:?}" )) })?; @@ -298,6 +482,9 @@ pub async fn handle_rust_job( let bin_path = format!("{}/{hash}", RUST_CACHE_DIR); let remote_path = format!("{RUST_OBJECT_STORE_PREFIX}{hash}"); + let reserved_variables = + get_reserved_variables(job, &client.token, conn, parent_runnable_path).await?; + let (cache, cache_logs) = windmill_common::worker::load_cache(&bin_path, &remote_path, false).await; @@ -332,16 +519,16 @@ pub async fn handle_rust_job( create_args_and_out_file(client, job, job_dir, conn).await?; build_rust_crate( - &job.id, + &job, mem_peak, canceled_by, job_dir, conn, worker_name, - &job.workspace_id, base_internal_url, &hash, occupancy_metrics, + requirements_o.is_none(), ) .await? }; @@ -349,9 +536,6 @@ pub async fn handle_rust_job( let logs2 = format!("{cache_logs}\n\n--- RUST CODE EXECUTION ---\n"); append_logs(&job.id, &job.workspace_id, logs2, conn).await; - let reserved_variables = - get_reserved_variables(job, &client.token, conn, parent_runnable_path).await?; - let child = if !*DISABLE_NSJAIL { let _ = write_file( job_dir, @@ -361,6 +545,7 @@ pub async fn handle_rust_job( .replace("{CACHE_DIR}", RUST_CACHE_DIR) .replace("{CACHE_HASH}", &hash) .replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string()) + .replace("{DEV}", DEV_CONF_NSJAIL) .replace("{SHARED_MOUNT}", shared_mount), )?; let mut nsjail_cmd = Command::new(NSJAIL_PATH.as_str()); diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index e15c24ef70..56ac7c00e0 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -18,23 +18,23 @@ use windmill_common::{ cache::{future::FutureCachedExt, ScriptData, ScriptMetadata}, schema::{should_validate_schema, SchemaValidator}, scripts::PREVIEW_IS_TAR_CODEBASE_HASH, - utils::WarnAfterExt, + utils::{create_directory_async, WarnAfterExt}, worker::{ - write_file, Connection, HttpClient, MAX_TIMEOUT, ROOT_CACHE_DIR, ROOT_CACHE_NOMOUNT_DIR, - TMP_DIR, + make_pull_query, write_file, Connection, HttpClient, MAX_TIMEOUT, ROOT_CACHE_DIR, + ROOT_CACHE_NOMOUNT_DIR, TMP_DIR, }, KillpillSender, }; #[cfg(feature = "enterprise")] -use windmill_common::ee::LICENSE_KEY_VALID; +use windmill_common::ee_oss::LICENSE_KEY_VALID; use anyhow::Result; use const_format::concatcp; #[cfg(feature = "prometheus")] use prometheus::IntCounter; -use tracing::{field, Instrument}; +use tracing::{field, Instrument, Span}; #[cfg(feature = "prometheus")] use windmill_common::METRICS_DEBUG_ENABLED; #[cfg(feature = "prometheus")] @@ -44,7 +44,7 @@ use serde::{Deserialize, Serialize}; use sqlx::types::Json; use std::{ collections::HashMap, - fs::DirBuilder, + fmt::Display, sync::{ atomic::{AtomicBool, AtomicU16, Ordering}, Arc, @@ -168,15 +168,10 @@ use windmill_common::bench::{benchmark_init, BenchmarkInfo, BenchmarkIter}; use windmill_common::add_time; -pub const PY310_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "python_310"); -pub const PY311_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "python_311"); -pub const PY312_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "python_312"); -pub const PY313_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "python_313"); - -pub const TAR_PY310_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "tar/python_310"); -pub const TAR_PY311_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "tar/python_311"); -pub const TAR_PY312_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "tar/python_312"); -pub const TAR_PY313_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "tar/python_313"); +pub const PY310_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "python_3_10"); +pub const PY311_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "python_3_11"); +pub const PY312_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "python_3_12"); +pub const PY313_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "python_3_13"); pub const TAR_JAVA_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "tar/java"); @@ -210,6 +205,9 @@ const NUM_SECS_READINGS: u64 = 60; const INCLUDE_DEPS_PY_SH_CONTENT: &str = include_str!("../nsjail/download_deps.py.sh"); +const WORKER_SHELL_NAP_TIME_DURATION: u64 = 15; +const TIMEOUT_TO_RESET_WORKER_SHELL_NAP_TIME_DURATION: u64 = 2 * 60; + pub const DEFAULT_SLEEP_QUEUE: u64 = 50; // only 1 native job so that we don't have to worry about concurrency issues on non dedicated native jobs workers @@ -247,6 +245,8 @@ lazy_static::lazy_static! { const DOTNET_DEFAULT_PATH: &str = "C:\\Program Files\\dotnet\\dotnet.exe"; #[cfg(unix)] const DOTNET_DEFAULT_PATH: &str = "/usr/bin/dotnet"; +pub const SAME_WORKER_REQUIREMENTS: &'static str = + "SameWorkerSender is required because this job may be part of a flow"; lazy_static::lazy_static! { @@ -391,6 +391,31 @@ lazy_static::lazy_static! { pub static ref WIN_ENVS: Envs = vec![]; } +#[derive(Debug)] +pub enum NextJob { + Sql(PulledJob), + Http(JobAndPerms), +} + +impl NextJob { + pub fn job(self) -> MiniPulledJob { + match self { + NextJob::Sql(job) => job.job, + NextJob::Http(job) => job.job, + } + } +} + +impl std::ops::Deref for NextJob { + type Target = MiniPulledJob; + fn deref(&self) -> &Self::Target { + match self { + NextJob::Sql(job) => &job.job, + NextJob::Http(job) => &job.job, + } + } +} + //only matter if CLOUD_HOSTED pub const MAX_RESULT_SIZE: usize = 1024 * 1024 * 2; // 2MB @@ -431,20 +456,26 @@ impl JobCompletedReceiver { } impl JobCompletedSender { + pub fn new_job_completed_sender_sql(buffer_size: u8) -> (Self, JobCompletedReceiver) { + let (sender, receiver) = flume::bounded::(buffer_size as usize); + let (unbounded_sender, unbounded_rx) = flume::unbounded::(); + let (killpill_tx, killpill_rx) = broadcast::channel::<()>(10); + ( + Self::Sql(SqlJobCompletedSender { sender, unbounded_sender, killpill_tx }), + JobCompletedReceiver { bounded_rx: receiver, killpill_rx, unbounded_rx }, + ) + } + pub fn new(conn: &Connection, buffer_size: u8) -> (Self, Option) { match conn { Connection::Sql(_) => { - let (sender, receiver) = flume::bounded::(buffer_size as usize); - let (unbounded_sender, unbounded_rx) = flume::unbounded::(); - let (killpill_tx, killpill_rx) = broadcast::channel::<()>(10); - ( - Self::Sql(SqlJobCompletedSender { sender, unbounded_sender, killpill_tx }), - Some(JobCompletedReceiver { bounded_rx: receiver, killpill_rx, unbounded_rx }), - ) + let result = Self::new_job_completed_sender_sql(buffer_size); + (result.0, Some(result.1)) } Connection::Http(client) => (Self::Http(client.clone()), None), } } + pub fn new_never_used() -> (Self, Option>) { (Self::NeverUsed, None) } @@ -622,6 +653,233 @@ fn add_outstanding_wait_time( } } +async fn extract_job_and_perms(job: NextJob, conn: &Connection) -> JobAndPerms { + match (job, conn) { + (NextJob::Sql(job), Connection::Sql(db)) => job.get_job_and_perms(db).await, + (NextJob::Sql(_), Connection::Http(_)) => panic!("sql job on http connection"), + (NextJob::Http(job), _) => job, + } +} + +fn create_span(arc_job: &Arc, worker_name: &str, hostname: &str) -> Span { + let span = tracing::span!(tracing::Level::INFO, "job", + job_id = %arc_job.id, root_job = field::Empty, workspace_id = %arc_job.workspace_id, worker = %worker_name, hostname = %hostname, tag = %arc_job.tag, + language = field::Empty, + script_path = field::Empty, flow_step_id = field::Empty, parent_job = field::Empty, + otel.name = field::Empty); + + let rj = arc_job.flow_innermost_root_job.unwrap_or(arc_job.id); + + if let Some(lg) = arc_job.script_lang.as_ref() { + span.record("language", lg.as_str()); + } + if let Some(step_id) = arc_job.flow_step_id.as_ref() { + span.record("otel.name", format!("job {}", step_id).as_str()); + span.record("flow_step_id", step_id.as_str()); + } else { + span.record("otel.name", "job"); + } + if let Some(parent_job) = arc_job.parent_job.as_ref() { + span.record("parent_job", parent_job.to_string().as_str()); + } + if let Some(script_path) = arc_job.runnable_path.as_ref() { + span.record("script_path", script_path.as_str()); + } + if let Some(root_job) = arc_job.flow_innermost_root_job.as_ref() { + span.record("root_job", root_job.to_string().as_str()); + } + + windmill_common::otel_oss::set_span_parent(&span, &rj); + span +} + +pub async fn handle_all_job_kind_error( + conn: &Connection, + authed_client: &AuthedClient, + job: Arc, + err: Error, + same_worker_tx: Option<&SameWorkerSender>, + worker_dir: &str, + worker_name: &str, + job_completed_tx: JobCompletedSender, + #[cfg(feature = "benchmark")] bench: &mut BenchmarkIter, +) { + match conn { + Connection::Sql(db) => { + handle_job_error( + db, + authed_client, + job.as_ref(), + 0, + None, + err, + false, + same_worker_tx, + &worker_dir, + &worker_name, + job_completed_tx.clone(), + #[cfg(feature = "benchmark")] + bench, + ) + .await; + } + Connection::Http(_) => { + job_completed_tx + .send_job( + JobCompleted { + preprocessed_args: None, + job: job.clone(), + result: Arc::new(windmill_common::worker::to_raw_value(&error_to_value( + err, + ))), + result_columns: None, + mem_peak: 0, + canceled_by: None, + success: false, + cached_res_path: None, + token: authed_client.token.clone(), + duration: None, + }, + false, + ) + .await + .expect("send job completed"); + } + } +} + +pub fn start_interactive_worker_shell( + conn: Connection, + hostname: String, + worker_name: String, + mut killpill_rx: tokio::sync::broadcast::Receiver<()>, + job_completed_tx: JobCompletedSender, + base_internal_url: String, + worker_dir: String, +) -> JoinHandle<()> { + tokio::spawn(async move { + let mut occupancy_metrics = OccupancyMetrics::new(Instant::now()); + + let mut last_executed_job: Option = + Instant::now().checked_sub(Duration::from_millis(2500)); + + loop { + if let Ok(_) = killpill_rx.try_recv() { + break; + } else { + let pulled_job = match &conn { + Connection::Sql(db) => { + let query = ("".to_string(), make_pull_query(&[hostname.to_owned()])); + + #[cfg(feature = "benchmark")] + let mut bench = windmill_common::bench::BenchmarkIter::new(); + let job = pull( + &db, + false, + &worker_name, + Some(&query), + #[cfg(feature = "benchmark")] + &mut bench, + ) + .await; + + job.map(|x| x.job.map(NextJob::Sql)) + } + Connection::Http(client) => { + crate::agent_workers::pull_job(&client, None, Some(true)) + .await + .map_err(|e| error::Error::InternalErr(e.to_string())) + .map(|x| x.map(|y| NextJob::Http(y))) + } + }; + + match pulled_job { + Ok(Some(job)) => { + tracing::debug!(worker = %worker_name, hostname = %hostname, "started handling of job {}", job.id); + + let job_dir = create_job_dir(&worker_dir, job.id).await; + #[cfg(feature = "benchmark")] + let mut bench = windmill_common::bench::BenchmarkIter::new(); + + let JobAndPerms { + job, + raw_code, + raw_lock, + raw_flow, + parent_runnable_path, + token, + precomputed_agent_info: precomputed_bundle, + } = extract_job_and_perms(job, &conn).await; + + let authed_client = AuthedClient::new( + base_internal_url.to_owned(), + job.workspace_id.clone(), + token, + None, + ); + + let arc_job = Arc::new(job); + + let _ = handle_queued_job( + arc_job.clone(), + raw_code, + raw_lock, + raw_flow, + parent_runnable_path, + &conn, + &authed_client, + &hostname, + &worker_name, + &worker_dir, + &job_dir, + None, + &base_internal_url, + job_completed_tx.clone(), + &mut occupancy_metrics, + &mut killpill_rx, + precomputed_bundle, + #[cfg(feature = "benchmark")] + &mut bench, + ) + .await; + + last_executed_job = Some(Instant::now()); + } + Ok(None) => { + let now = Instant::now(); + match last_executed_job { + Some(last) + if now.duration_since(last).as_secs() + > TIMEOUT_TO_RESET_WORKER_SHELL_NAP_TIME_DURATION => + { + tokio::time::sleep(Duration::from_secs( + WORKER_SHELL_NAP_TIME_DURATION, + )) + .await; + } + _ => { + tokio::time::sleep(Duration::from_millis(*SLEEP_QUEUE)).await; + } + } + } + + Err(err) => { + tracing::error!(worker = %worker_name, hostname = %hostname, "Failed to pull jobs: {}", err); + } + }; + } + } + }) +} + +async fn create_job_dir(worker_directory: &str, job_id: impl Display) -> String { + let job_dir_path = format!("{}/{}", worker_directory, job_id); + + create_directory_async(&job_dir_path).await; + + job_dir_path +} + pub async fn run_worker( conn: &Connection, hostname: &str, @@ -686,10 +944,7 @@ pub async fn run_worker( write_file(&HOME_ENV, ".netrc", netrc).expect("could not write netrc"); } - DirBuilder::new() - .recursive(true) - .create(&worker_dir) - .expect("could not create initial worker dir"); + create_directory_async(&worker_dir).await; if !*DISABLE_NSJAIL { let _ = write_file( @@ -948,6 +1203,25 @@ pub async fn run_worker( )), _ => None, }; + // If we're the first worker to run, we start another background process that listens for a specific tag. + // This tag is associated only with jobs using Bash as the script language. + // For agent workers, the expected tag format is the worker name suffixed with "-ssh". + // For regular workers, the tag is simply the machine's hostname and if not found the randomly generated hostname. + let interactive_shell = if i_worker == 1 { + let it_shell = start_interactive_worker_shell( + conn.clone(), + hostname.to_owned(), + worker_name.clone(), + killpill_rx.resubscribe(), + job_completed_tx.clone(), + base_internal_url.to_owned(), + worker_dir.clone(), + ); + + Some(it_shell) + } else { + None + }; let mut last_executed_job: Option = None; @@ -1058,7 +1332,7 @@ pub async fn run_worker( if !valid_key { tracing::error!( worker = %worker_name, hostname = %hostname, - "Invalid license key, workers require a valid license key, sleeping for 30s waiting for valid key to be set" + "Invalid license key, workers require a valid license key, sleeping for 10s waiting for valid key to be set" ); tokio::time::sleep(Duration::from_secs(10)).await; continue; @@ -1128,29 +1402,6 @@ pub async fn run_worker( } else { tracing::info!("benchmark not finished, still pulling jobs {}", infos.iters); } - enum NextJob { - Sql(PulledJob), - Http(JobAndPerms), - } - - impl NextJob { - pub fn job(self) -> MiniPulledJob { - match self { - NextJob::Sql(job) => job.job, - NextJob::Http(job) => job.job, - } - } - } - - impl std::ops::Deref for NextJob { - type Target = MiniPulledJob; - fn deref(&self) -> &Self::Target { - match self { - NextJob::Sql(job) => &job.job, - NextJob::Http(job) => &job.job, - } - } - } let next_job = { // println!("2: {:?}", instant.elapsed()); @@ -1191,6 +1442,7 @@ pub async fn run_worker( "/api/agent_workers/same_worker_job/{}", same_worker_job.job_id ), + None, &same_worker_job, ) .await @@ -1296,7 +1548,7 @@ pub async fn run_worker( } job.map(|x| x.job.map(NextJob::Sql)) } - Connection::Http(client) => crate::agent_workers::pull_job(&client) + Connection::Http(client) => crate::agent_workers::pull_job(&client, None, None) .await .map_err(|e| error::Error::InternalErr(e.to_string())) .map(|x| x.map(|y| NextJob::Http(y))), @@ -1344,6 +1596,7 @@ pub async fn run_worker( } } } + if matches!(job.kind, JobKind::Noop) { add_time!(bench, "send job completed START"); job_completed_tx @@ -1425,20 +1678,12 @@ pub async fn run_worker( // fields macro we can make a job id that only appears when // the job is defined? - let job_dir = format!("{worker_dir}/{}", job.id); - - DirBuilder::new() - .recursive(true) - .create(&job_dir) - .expect("could not create job dir"); + let job_dir = create_job_dir(&worker_dir, job.id).await; let same_worker = job.same_worker; let folder = if job.script_lang == Some(ScriptLang::Go) { - DirBuilder::new() - .recursive(true) - .create(&format!("{job_dir}/go")) - .expect("could not create go dir"); + create_directory_async(&format!("{job_dir}/go")).await; "/go" } else { "" @@ -1450,26 +1695,22 @@ pub async fn run_worker( if tokio::fs::metadata(target).await.is_err() { let parent_flow = job.parent_job.unwrap(); let parent_shared_dir = format!("{worker_dir}/{parent_flow}/shared"); - DirBuilder::new() - .recursive(true) - .create(&parent_shared_dir) - .expect("could not create parent shared dir"); - + create_directory_async(&parent_shared_dir).await; symlink(&parent_shared_dir, target) .await .expect("could not symlink target"); } } else { - DirBuilder::new() - .recursive(true) - .create(target) - .expect("could not create shared dir"); + create_directory_async(target).await; } #[cfg(feature = "prometheus")] let tag = job.tag.clone(); let is_init_script: bool = job.tag.as_str() == INIT_SCRIPT_TAG; + let is_flow = job.is_flow(); + let job_id = job.id; + let JobAndPerms { job, raw_code, @@ -1478,70 +1719,32 @@ pub async fn run_worker( parent_runnable_path, token, precomputed_agent_info: precomputed_bundle, - } = match (job, &conn) { - (NextJob::Sql(job), Connection::Sql(db)) => job.get_job_and_perms(db).await, - (NextJob::Sql(_), Connection::Http(_)) => { - panic!("sql job on http connection") - } - (NextJob::Http(job), _) => job, - }; + } = extract_job_and_perms(job, &conn).await; - // let token = create_token(&db, &job, job_perms).await; - let authed_client = AuthedClient { - base_internal_url: base_internal_url.to_string(), + let authed_client = AuthedClient::new( + base_internal_url.to_owned(), + job.workspace_id.clone(), token, - workspace: job.workspace_id.to_string(), - force_client: None, - }; + None, + ); let arc_job = Arc::new(job); - add_time!(bench, "handle_queued_job START"); - let span = tracing::span!(tracing::Level::INFO, "job", - job_id = %arc_job.id, root_job = field::Empty, workspace_id = %arc_job.workspace_id, worker = %worker_name, hostname = %hostname, tag = %arc_job.tag, - language = field::Empty, - script_path = field::Empty, flow_step_id = field::Empty, parent_job = field::Empty, - otel.name = field::Empty); - let rj = if let Some(root_job) = arc_job.flow_innermost_root_job { - root_job - } else { - arc_job.id - }; - if let Some(lg) = arc_job.script_lang.as_ref() { - span.record("language", lg.as_str()); - } - if let Some(step_id) = arc_job.flow_step_id.as_ref() { - span.record("otel.name", format!("job {}", step_id).as_str()); - span.record("flow_step_id", step_id.as_str()); - } else { - span.record("otel.name", "job"); - } - if let Some(parent_job) = arc_job.parent_job.as_ref() { - span.record("parent_job", parent_job.to_string().as_str()); - } - if let Some(script_path) = arc_job.runnable_path.as_ref() { - span.record("script_path", script_path.as_str()); - } - if let Some(root_job) = arc_job.flow_innermost_root_job.as_ref() { - span.record("root_job", root_job.to_string().as_str()); - } + let span = create_span(&arc_job, &worker_name, hostname); - windmill_common::otel_ee::set_span_parent(&span, &rj); - // span.context().span().add_event_with_timestamp("job created".to_string(), arc_job.created_at.into(), vec![]); - - match handle_queued_job( + let job_result = handle_queued_job( arc_job.clone(), raw_code, raw_lock, raw_flow, parent_runnable_path, - conn, + &conn, &authed_client, - &hostname, + hostname, &worker_name, &worker_dir, &job_dir, - same_worker_tx.clone(), + Some(same_worker_tx.clone()), base_internal_url, job_completed_tx.clone(), &mut occupancy_metrics, @@ -1551,74 +1754,40 @@ pub async fn run_worker( &mut bench, ) .instrument(span) - .await - { + .await; + + match job_result { + Ok(false) if is_init_script => { + tracing::error!("init script job failed, exiting"); + update_worker_ping_for_failed_init_script(conn, &worker_name, job_id) + .await; + break; + } Err(err) => { - match conn { - Connection::Sql(db) => { - handle_job_error( - db, - &authed_client, - arc_job.as_ref(), - 0, - None, - err, - false, - same_worker_tx.clone(), - &worker_dir, - &worker_name, - job_completed_tx.clone(), - #[cfg(feature = "benchmark")] - &mut bench, - ) - .await; - } - Connection::Http(_) => { - job_completed_tx - .send_job( - JobCompleted { - preprocessed_args: None, - job: arc_job.clone(), - result: Arc::new( - windmill_common::worker::to_raw_value( - &error_to_value(err), - ), - ), - result_columns: None, - mem_peak: 0, - canceled_by: None, - success: false, - cached_res_path: None, - token: authed_client.token.clone(), - duration: None, - }, - false, - ) - .await - .expect("send job completed"); - } - } + handle_all_job_kind_error( + &conn, + &authed_client, + arc_job.clone(), + err, + Some(&same_worker_tx), + &worker_dir, + &worker_name, + job_completed_tx.clone(), + #[cfg(feature = "benchmark")] + &mut bench, + ) + .await; if is_init_script { tracing::error!("init script job failed (in handler), exiting"); update_worker_ping_for_failed_init_script( conn, &worker_name, - arc_job.id, + job_id, ) .await; break; } } - Ok(false) if is_init_script => { - tracing::error!("init script job failed, exiting"); - update_worker_ping_for_failed_init_script( - conn, - &worker_name, - arc_job.id, - ) - .await; - break; - } _ => {} } @@ -1643,8 +1812,7 @@ pub async fn run_worker( .await; } - if !KEEP_JOB_DIR.load(Ordering::Relaxed) && !(arc_job.is_flow() && same_worker) - { + if !KEEP_JOB_DIR.load(Ordering::Relaxed) && !(is_flow && same_worker) { let _ = tokio::fs::remove_dir_all(job_dir).await; } } @@ -1726,6 +1894,11 @@ pub async fn run_worker( tracing::error!("error in awaiting send_result process: {e:?}") } } + if let Some(interactive_shell) = interactive_shell { + if let Err(e) = interactive_shell.await { + tracing::error!("error in awaiting interactive_shell process: {e:?}") + } + } tracing::info!(worker = %worker_name, hostname = %hostname, "worker {} exited", worker_name); tracing::info!(worker = %worker_name, hostname = %hostname, "number of jobs executed: {}", jobs_executed); } @@ -1814,7 +1987,7 @@ pub struct PreviousResult<'a> { pub previous_result: Option<&'a RawValue>, } -async fn handle_queued_job( +pub async fn handle_queued_job( job: Arc, raw_code: Option, raw_lock: Option, @@ -1826,7 +1999,7 @@ async fn handle_queued_job( worker_name: &str, worker_dir: &str, job_dir: &str, - same_worker_tx: SameWorkerSender, + same_worker_tx: Option, base_internal_url: &str, job_completed_tx: JobCompletedSender, occupancy_metrics: &mut OccupancyMetrics, @@ -1939,7 +2112,7 @@ async fn handle_queued_job( .to_string(); append_logs(&job.id, &job.workspace_id, logs, conn).await; } - job_completed_tx + let result = job_completed_tx .send_job( JobCompleted { preprocessed_args: None, @@ -1955,8 +2128,16 @@ async fn handle_queued_job( }, true, ) - .await - .expect("send job completed"); + .await; + + match result { + Ok(_) => { + tracing::debug!("Send job completed") + } + Err(err) => { + tracing::error!("An error occurred while sending job completed: {:#?}", err) + } + } return Ok(true); } @@ -1975,7 +2156,7 @@ async fn handle_queued_job( db, &client, None, - same_worker_tx, + &same_worker_tx.expect(SAME_WORKER_REQUIREMENTS), worker_dir, job_completed_tx.clone(), worker_name, @@ -2144,7 +2325,6 @@ async fn handle_queued_job( { return Ok(false); } - process_result( job, result.map(|x| Arc::new(x)), diff --git a/backend/windmill-worker/src/worker_flow.rs b/backend/windmill-worker/src/worker_flow.rs index 6c0b133828..50773589da 100644 --- a/backend/windmill-worker/src/worker_flow.rs +++ b/backend/windmill-worker/src/worker_flow.rs @@ -66,7 +66,7 @@ use windmill_queue::{ type DB = sqlx::Pool; -use windmill_audit::audit_ee::{audit_log, AuditAuthor}; +use windmill_audit::audit_oss::{audit_log, AuditAuthor}; use windmill_audit::ActionKind; use windmill_queue::{canceled_job_to_result, push}; @@ -80,7 +80,7 @@ pub async fn update_flow_status_after_job_completion( success: bool, result: Arc>, unrecoverable: bool, - same_worker_tx: SameWorkerSender, + same_worker_tx: &SameWorkerSender, worker_dir: &str, stop_early_override: Option, worker_name: &str, @@ -89,6 +89,7 @@ pub async fn update_flow_status_after_job_completion( ) -> error::Result>> { // this is manual tailrecursion because async_recursion blows up the stack potentially_crash_for_testing(); + let mut rec = RecUpdateFlowStatusAfterJobCompletion { flow, job_id_for_status: job_id_for_status.clone(), @@ -109,7 +110,7 @@ pub async fn update_flow_status_after_job_completion( rec.success, rec.result, unrecoverable, - same_worker_tx.clone(), + same_worker_tx, worker_dir, rec.stop_early_override, rec.skip_error_handler, @@ -134,7 +135,7 @@ pub async fn update_flow_status_after_job_completion( error: json!(e.to_string()), }))), true, - same_worker_tx.clone(), + same_worker_tx, worker_dir, rec.stop_early_override, rec.skip_error_handler, @@ -147,6 +148,7 @@ pub async fn update_flow_status_after_job_completion( } }; unrecoverable = false; + match nrec { UpdateFlowStatusAfterJobCompletion::Done(job) => { add_time!(bench, "update flow status internal END"); @@ -226,7 +228,7 @@ pub async fn update_flow_status_after_job_completion_internal( mut success: bool, result: Arc>, unrecoverable: bool, - same_worker_tx: SameWorkerSender, + same_worker_tx: &SameWorkerSender, worker_dir: &str, stop_early_override: Option, skip_error_handler: bool, @@ -625,9 +627,10 @@ pub async fn update_flow_status_after_job_completion_internal( sqlx::query!( "UPDATE v2_job_queue q SET suspend = 0 FROM v2_job j, v2_job_status f - WHERE parent_job = $1 + WHERE q.workspace_id = $1 AND q.suspend = $3 AND j.parent_job = $2 AND f.id = j.id AND q.id = j.id - AND suspend = $2 AND (f.flow_status->'step')::int = 0", + AND (f.flow_status->'step')::int = 0", + w_id, flow, nindex ) @@ -1289,7 +1292,7 @@ pub async fn update_flow_status_after_job_completion_internal( db, client, Some(nresult.clone()), - same_worker_tx.clone(), + same_worker_tx, worker_dir, job_completed_tx, worker_name, @@ -1586,7 +1589,7 @@ pub async fn handle_flow( db: &sqlx::Pool, client: &AuthedClient, last_result: Option>>, - same_worker_tx: SameWorkerSender, + same_worker_tx: &SameWorkerSender, worker_dir: &str, job_completed_tx: JobCompletedSender, worker_name: &str, @@ -1643,7 +1646,7 @@ pub async fn handle_flow( db, client, last_result.clone(), - same_worker_tx.clone(), + same_worker_tx, worker_dir, worker_name, ) @@ -1736,7 +1739,7 @@ async fn push_next_flow_job( db: &sqlx::Pool, client: &AuthedClient, last_job_result: Option>>, - same_worker_tx: SameWorkerSender, + same_worker_tx: &SameWorkerSender, worker_dir: &str, worker_name: &str, ) -> error::Result { diff --git a/backend/windmill-worker/src/worker_lockfiles.rs b/backend/windmill-worker/src/worker_lockfiles.rs index 6710f718c4..33c0375f09 100644 --- a/backend/windmill-worker/src/worker_lockfiles.rs +++ b/backend/windmill-worker/src/worker_lockfiles.rs @@ -1965,12 +1965,11 @@ async fn ansible_dep( use windmill_parser_yaml::add_versions_to_requirements_yaml; use crate::ansible_executor::{ - create_ansible_cfg, get_collection_locks, get_git_ssh_cmd, get_role_locks, - install_galaxy_collections, - }; + create_ansible_cfg, get_collection_locks, get_git_ssh_cmd, get_role_locks, + install_galaxy_collections, + }; use windmill_common::client::AuthedClient; - let python_lockfile = python_dep( reqs.python_reqs.join("\n").to_string(), job_id, @@ -1989,12 +1988,12 @@ async fn ansible_dep( let conn = &Connection::Sql(db.clone()); - let authed_client = AuthedClient { - base_internal_url: base_internal_url.to_string(), - token: token.to_string(), - workspace: w_id.to_string(), - force_client: None, - }; + let authed_client = AuthedClient::new( + base_internal_url.to_string(), + w_id.to_string(), + token.to_string(), + None, + ); let git_ssh_cmd = get_git_ssh_cmd(&reqs, job_dir, &authed_client).await?; @@ -2102,7 +2101,12 @@ async fn capture_dependency_job( ( job_raw_code.to_owned(), - crate::PyV::parse_from_requirements(&split_requirements(job_raw_code)), + match crate::PyV::try_parse_from_requirements(&split_requirements( + job_raw_code, + )) { + Some(pyv) => pyv, + None => crate::PyV::gravitational_version(job_id, w_id, None).await, + }, ) } else { let mut version_specifiers = vec![]; diff --git a/backend/windmill-worker/src/worker_utils.rs b/backend/windmill-worker/src/worker_utils.rs index e1ea849999..4da318d37b 100644 --- a/backend/windmill-worker/src/worker_utils.rs +++ b/backend/windmill-worker/src/worker_utils.rs @@ -125,6 +125,7 @@ async fn update_worker_ping_full_inner( client .post::<_, ()>( UPDATE_PING_URL, + None, &Ping { last_job_executed: None, last_job_workspace_id: None, @@ -190,6 +191,7 @@ pub async fn insert_ping( client .post::<_, ()>( UPDATE_PING_URL, + None, &Ping { last_job_executed: None, last_job_workspace_id: None, @@ -249,6 +251,7 @@ pub async fn update_worker_ping_from_job( client .post::( UPDATE_PING_URL, + None, &Ping { last_job_executed: Some(job_id.clone()), last_job_workspace_id: Some(w_id.to_string()), @@ -287,6 +290,7 @@ pub async fn ping_job_status( client .post( &format!("/api/agent_workers/ping_job_status/{}", job_id), + None, &PingJobStatus { mem_peak, current_mem }, ) .await diff --git a/benchmarks/lib.ts b/benchmarks/lib.ts index 3ff3ab48d5..27413cc466 100644 --- a/benchmarks/lib.ts +++ b/benchmarks/lib.ts @@ -2,7 +2,7 @@ import { sleep } from "https://deno.land/x/sleep@v1.2.1/mod.ts"; import * as windmill from "https://deno.land/x/windmill@v1.174.0/mod.ts"; import * as api from "https://deno.land/x/windmill@v1.174.0/windmill-api/index.ts"; -export const VERSION = "v1.493.2"; +export const VERSION = "v1.496.3"; export async function login(email: string, password: string): Promise { return await windmill.UserService.login({ diff --git a/cli/main.ts b/cli/main.ts index f2d53e07ff..50da78522d 100644 --- a/cli/main.ts +++ b/cli/main.ts @@ -63,7 +63,7 @@ export { // } // }); -export const VERSION = "1.493.2"; +export const VERSION = "1.496.3"; const command = new Command() .name("wmill") diff --git a/docker/DockerfileFull b/docker/DockerfileFull index 78eec753d1..ddc1cdb3af 100644 --- a/docker/DockerfileFull +++ b/docker/DockerfileFull @@ -3,6 +3,7 @@ FROM ghcr.io/windmill-labs/windmill:dev # Rust COPY --from=rust:1.86.0 /usr/local/cargo /usr/local/cargo COPY --from=rust:1.86.0 /usr/local/rustup /usr/local/rustup +RUN /usr/local/cargo/bin/cargo install cargo-sweep --version ^0.7 # Ansible RUN uv tool install ansible && [ -d "$(uv tool dir)/ansible/bin/" ] && find "$(uv tool dir)/ansible/bin/" -mindepth 1 -maxdepth 1 -type f -executable -regextype posix-extended -regex '^((.+/)?)[^.]+' -print0 | xargs -0 ln -s -t "$UV_TOOL_BIN_DIR/" || true diff --git a/docker/DockerfileFullEe b/docker/DockerfileFullEe index 138038559c..7ed7108de2 100644 --- a/docker/DockerfileFullEe +++ b/docker/DockerfileFullEe @@ -22,6 +22,7 @@ FROM ghcr.io/windmill-labs/windmill-ee:dev # Rust COPY --from=rust:1.86.0 /usr/local/cargo /usr/local/cargo COPY --from=rust:1.86.0 /usr/local/rustup /usr/local/rustup +RUN /usr/local/cargo/bin/cargo install cargo-sweep --version ^0.7 # Ansible RUN uv tool install ansible && [ -d "$(uv tool dir)/ansible/bin/" ] && find "$(uv tool dir)/ansible/bin/" -mindepth 1 -maxdepth 1 -type f -executable -regextype posix-extended -regex '^((.+/)?)[^.]+' -print0 | xargs -0 ln -s -t "$UV_TOOL_BIN_DIR/" || true diff --git a/docker/DockerfileNsjail b/docker/DockerfileNsjail index e51e73a3ea..cd17557256 100644 --- a/docker/DockerfileNsjail +++ b/docker/DockerfileNsjail @@ -45,6 +45,7 @@ RUN apt-get update && apt-get install -y libprotobuf-dev libnl-route-3-dev # Rust COPY --from=rust:1.80.1 /usr/local/cargo /usr/local/cargo COPY --from=rust:1.80.1 /usr/local/rustup /usr/local/rustup +RUN /usr/local/cargo/bin/cargo install cargo-sweep --version ^0.7 # Ansible RUN uv tool install ansible && [ -d "$(uv tool dir)/ansible/bin/" ] && find "$(uv tool dir)/ansible/bin/" -mindepth 1 -maxdepth 1 -type f -executable -regextype posix-extended -regex '^((.+/)?)[^.]+' -print0 | xargs -0 ln -s -t "$UV_TOOL_BIN_DIR/" || true diff --git a/flake.nix b/flake.nix index f9df73f465..d11b967cc4 100644 --- a/flake.nix +++ b/flake.nix @@ -81,17 +81,22 @@ buildInputs = buildInputs ++ (with pkgs; [ # Essentials rust + cargo-sweep git xcaddy sqlx-cli sccache nsjail + openapi-generator-cli # Python flock python3 python3Packages.pip uv + poetry + pyright + openapi-python-client # Other languages deno @@ -186,6 +191,7 @@ # for related places search: ADD_NEW_LANG FLOCK_PATH = "${pkgs.flock}/bin/flock"; CARGO_PATH = "${rust}/bin/cargo"; + CARGO_SWEEP_PATH = "${pkgs.cargo-sweep}/bin/cargo-sweep"; DOTNET_PATH = "${pkgs.dotnet-sdk_9}/bin/dotnet"; DOTNET_ROOT = "${pkgs.dotnet-sdk_9}/share/dotnet"; ORACLE_LIB_DIR = "${pkgs.oracle-instantclient.lib}/lib"; diff --git a/frontend/CLAUDE.md b/frontend/CLAUDE.md deleted file mode 100644 index 63f35ab820..0000000000 --- a/frontend/CLAUDE.md +++ /dev/null @@ -1,241 +0,0 @@ -# Svelte 5 Best Practices - -This guide outlines best practices for developing with Svelte 5, incorporating the new Runes API and other modern Svelte features. These rules MUST NOT be applied on svelte 4 files unless explicitly asked to do so. - -## Reactivity with Runes - -Svelte 5 introduces Runes for more explicit and flexible reactivity. - -1. **Embrace Runes for State Management**: - - - Use `$state` for reactive local component state. - - ```svelte - - - - ``` - - - Use `$derived` for computed values based on other reactive state. - - ```svelte - - -

{count} * 2 = {doubled}

- ``` - - - Use `$effect` for side effects that need to run when reactive values change (e.g., logging, manual DOM manipulation, data fetching). Remember `$effect` does not run on the server. - - ```svelte - - ``` - -2. **Props with `$props`**: - - - Declare component props using `$props()`. This offers better clarity and flexibility compared to `export let`. - - ```svelte - - -

Name: {name}

-

Age: {age}

- ``` - - - For bindable props, use `$bindable`. - - ```svelte - - - - ``` - -## Event Handling - -- **Use direct event attributes**: Svelte 5 moves away from `on:` directives for DOM events. - - **Do**: `` - - **Don't**: `` -- **For component events, prefer callback props**: Instead of `createEventDispatcher`, pass functions as props. - - ```svelte - - - -

Message from child: {message}

- - - - - ``` - -## Snippets for Content Projection - -- **Use `{#snippet ...}` and `{@render ...}` instead of slots**: Snippets are more powerful and flexible. - - ```svelte - - - - - {#snippet title()} - My Awesome Title - {/snippet} - {#snippet content()} -

Some interesting content here.

- {/snippet} -
- - - - -
-
{@render title()}
-
{@render content()}
-
- ``` - -- Default content is passed via the `children` prop (which is a snippet). - ```svelte - - -
- {@render children?.()} -
- ``` - -## Component Design - -1. **Create Small, Reusable Components**: Break down complex UIs into smaller, focused components. Each component should have a single responsibility. This also aids performance by limiting the scope of reactivity updates. -2. **Descriptive Naming**: Use clear and descriptive names for variables, functions, and components. -3. **Minimize Logic in Components**: Move complex business logic to utility functions or services. Keep components focused on presentation and interaction. - -## State Management (Stores) - -1. **Segment Stores**: Avoid a single global store. Create multiple stores, each responsible for a specific piece of global state (e.g., `userStore.js`, `themeStore.js`). This can help limit reactivity updates to only the parts of the UI that depend on specific state segments. -2. **Use Custom Stores for Complex Logic**: For stores with related methods, create custom stores. - - ```javascript - // counterStore.js - import { writable } from 'svelte/store' - - function createCounter() { - const { subscribe, set, update } = writable(0) - - return { - subscribe, - increment: () => update((n) => n + 1), - decrement: () => update((n) => n - 1), - reset: () => set(0) - } - } - export const counter = createCounter() - ``` - -3. **Use Context API for Localized State**: For state shared within a component subtree, consider Svelte's context API (`setContext`, `getContext`) instead of global stores when the state doesn't need to be truly global. - -## Performance Optimizations (Svelte 5) - -When generating Svelte 5 code, prioritize frontend performance by applying the following principles: - -### General Svelte 5 Principles - -- **Leverage the Compiler:** Trust Svelte's compiler to generate optimized JavaScript. Avoid manual DOM manipulation (`document.querySelector`, etc.) unless absolutely necessary for integrating third-party libraries that lack Svelte adapters. -- **Keep Components Small and Focused:** Reinforcing from Component Design, smaller components lead to less complex reactivity graphs and more targeted, efficient updates. - -### Reactivity & State Management - -- **Optimize Computations with `$derived`:** Always use `$derived` for computed values that depend on other state. This ensures the computation only runs when its specific dependencies change, avoiding unnecessary work compared to recomputing derived values in `$effect` or less efficient methods. -- **Minimize `$effect` Usage:** Use `$effect` sparingly and only for true side effects that interact with the outside world or non-Svelte state. Avoid putting complex logic or state updates _within_ an `$effect` unless those updates are explicitly intended as a reaction to external changes or non-Svelte state. Excessive or complex effects can impact rendering performance. -- **Structure State for Fine-Grained Updates:** Design your `$state` objects or variables such that updates affect only the necessary parts of the UI. Avoid putting too much unrelated state into a single large object that gets frequently updated, as this can potentially trigger broader updates than necessary. Consider normalizing complex, nested state. - -### List Rendering (`{#each}`) - -- **Mandate `key` Attribute:** Always use a `key` attribute (`{#each items as item (item.id)}`) that refers to a unique, stable identifier for each item in a list. This is critical for allowing Svelte to efficiently update, reorder, add, or remove list items without destroying and re-creating unnecessary DOM elements and component instances. - -### Component Loading & Bundling - -- **Implement Lazy Loading/Code Splitting:** For routes, components, or modules that are not immediately needed on page load, use dynamic imports (`import(...)`) to split the code bundle. SvelteKit handles this automatically for routes, but it can be applied manually to components using helper patterns if needed. -- **Be Mindful of Third-Party Libraries:** When incorporating external libraries, import only the necessary functions or components to minimize the final bundle size. Prefer libraries designed to be tree-shakeable. - -### Rendering & DOM - -- **Use CSS for Animations/Transitions:** Prefer CSS animations or transitions where possible for performance. Svelte's built-in `transition:` directive is also highly optimized and should be used for complex state-driven transitions, but simple cases can often use plain CSS. -- **Optimize Image Loading:** Implement best practices for images: use optimized formats (WebP, AVIF), lazy loading (`loading="lazy"`), and responsive images (``, `srcset`) to avoid loading unnecessarily large images. - -### Server-Side Rendering (SSR) & Hydration - -- **Ensure SSR Compatibility:** Write components that can be rendered on the server for faster initial page loads. Avoid relying on browser-specific APIs (like `window` or `document`) in the main ` - -
- Hello -
- ``` - -5. **Stay Updated**: Keep Svelte and its related packages up to date to benefit from the latest features, performance improvements, and security fixes. diff --git a/frontend/npm b/frontend/npm deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 16d3f6a057..511069631d 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -1,12 +1,13 @@ { "name": "windmill-components", - "version": "1.493.10", + "version": "1.496.3", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "windmill-components", - "version": "1.493.10", + "version": "1.496.3", + "hasInstallScript": true, "license": "AGPL-3.0", "dependencies": { "@aws-crypto/sha256-js": "^4.0.0", @@ -21,8 +22,10 @@ "@leeoniya/ufuzzy": "^1.0.8", "@popperjs/core": "^2.11.6", "@redocly/json-to-json-schema": "^0.0.1", + "@scalar/openapi-parser": "^0.15.0", "@tanstack/svelte-table": "npm:tanstack-table-8-svelte-5@^0.1", "@windmill-labs/svelte-dnd-action": "^0.9.48", + "@xterm/addon-fit": "^0.10.0", "@xyflow/svelte": "^0.1.15", "ag-charts-community": "^9.0.1", "ag-charts-enterprise": "^9.0.1", @@ -51,6 +54,7 @@ "monaco-vim": "^0.4.1", "ol": "^7.4.0", "openai": "^4.87.1", + "openapi-types": "^12.1.3", "p-limit": "^6.1.0", "panzoom": "^9.4.3", "pdfjs-dist": "4.8.69", @@ -78,11 +82,14 @@ "windmill-parser-wasm-ts": "^1.486.1", "windmill-parser-wasm-yaml": "^1.429.0", "windmill-sql-datatype-parser-wasm": "^1.318.0", + "xterm": "^5.3.0", + "xterm-readline": "^1.1.2", "y-monaco": "^0.1.4", "y-websocket": "^1.5.4", - "yaml": "^2.3.4", + "yaml": "^2.8.0", "yjs": "^13.6.7", - "zod": "^3.24.2" + "zod": "^3.24.2", + "zod-to-json-schema": "^3.24.5" }, "devDependencies": { "@floating-ui/core": "^1.3.1", @@ -144,7 +151,7 @@ "fsevents": "^2.3.3" }, "peerDependencies": { - "svelte": "^5.0.0" + "svelte": "^4.0.0" } }, "node_modules/@alloc/quick-lru": { @@ -2999,6 +3006,59 @@ "win32" ] }, + "node_modules/@scalar/openapi-parser": { + "version": "0.15.0", + "resolved": "https://registry.npmjs.org/@scalar/openapi-parser/-/openapi-parser-0.15.0.tgz", + "integrity": "sha512-49u1sE+P+72nbt/9p7flBS/g7wbapTGAplcQ1FZxa4La56OMuShCl4qSSdw8lPNZjUZNTLcw7ExVOGI9p1kkOA==", + "license": "MIT", + "dependencies": { + "ajv": "^8.17.1", + "ajv-draft-04": "^1.0.0", + "ajv-formats": "^3.0.1", + "jsonpointer": "^5.0.1", + "leven": "^4.0.0", + "yaml": "^2.4.5" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/@scalar/openapi-parser/node_modules/ajv": { + "version": "8.17.1", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.17.1.tgz", + "integrity": "sha512-B/gBuNg5SiMTrPkC+A2+cW0RszwxYmn6VYxB/inlBStS5nx6xHIt/ehKRhIMhqusl7a8LjQoZnjCs5vhwxOQ1g==", + "license": "MIT", + "dependencies": { + "fast-deep-equal": "^3.1.3", + "fast-uri": "^3.0.1", + "json-schema-traverse": "^1.0.0", + "require-from-string": "^2.0.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, + "node_modules/@scalar/openapi-parser/node_modules/ajv-draft-04": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/ajv-draft-04/-/ajv-draft-04-1.0.0.tgz", + "integrity": "sha512-mv00Te6nmYbRp5DCwclxtt7yV/joXJPGS7nM+97GdxvuttCOfgI3K4U25zboyeX0O+myI8ERluxQe5wljMmVIw==", + "license": "MIT", + "peerDependencies": { + "ajv": "^8.5.0" + }, + "peerDependenciesMeta": { + "ajv": { + "optional": true + } + } + }, + "node_modules/@scalar/openapi-parser/node_modules/json-schema-traverse": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", + "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==", + "license": "MIT" + }, "node_modules/@smithy/types": { "version": "4.2.0", "resolved": "https://registry.npmjs.org/@smithy/types/-/types-4.2.0.tgz", @@ -3794,6 +3854,22 @@ "svelte": ">=3.23.0 || ^5.0.0-next.0" } }, + "node_modules/@xterm/addon-fit": { + "version": "0.10.0", + "resolved": "https://registry.npmjs.org/@xterm/addon-fit/-/addon-fit-0.10.0.tgz", + "integrity": "sha512-UFYkDm4HUahf2lnEyHvio51TNGiLK66mqP2JoATy7hRZeXaGMRDr00JiSF7m63vR5WKATF605yEggJKsw0JpMQ==", + "license": "MIT", + "peerDependencies": { + "@xterm/xterm": "^5.0.0" + } + }, + "node_modules/@xterm/xterm": { + "version": "5.5.0", + "resolved": "https://registry.npmjs.org/@xterm/xterm/-/xterm-5.5.0.tgz", + "integrity": "sha512-hqJHYaQb5OptNunnyAnkHyM8aCjZ1MEIDTQu1iIbbTD/xops91NB5yq1ZK/dC2JDbVWtF23zUtl9JE2NqwT87A==", + "license": "MIT", + "peer": true + }, "node_modules/@xyflow/svelte": { "version": "0.1.39", "resolved": "https://registry.npmjs.org/@xyflow/svelte/-/svelte-0.1.39.tgz", @@ -3932,6 +4008,45 @@ "url": "https://github.com/sponsors/epoberezkin" } }, + "node_modules/ajv-formats": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/ajv-formats/-/ajv-formats-3.0.1.tgz", + "integrity": "sha512-8iUql50EUR+uUcdRQ3HDqa6EVyo3docL8g5WJ3FNcWmu62IbkGUue/pEyLBW8VGKKucTPgqeks4fIU1DA4yowQ==", + "license": "MIT", + "dependencies": { + "ajv": "^8.0.0" + }, + "peerDependencies": { + "ajv": "^8.0.0" + }, + "peerDependenciesMeta": { + "ajv": { + "optional": true + } + } + }, + "node_modules/ajv-formats/node_modules/ajv": { + "version": "8.17.1", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.17.1.tgz", + "integrity": "sha512-B/gBuNg5SiMTrPkC+A2+cW0RszwxYmn6VYxB/inlBStS5nx6xHIt/ehKRhIMhqusl7a8LjQoZnjCs5vhwxOQ1g==", + "license": "MIT", + "dependencies": { + "fast-deep-equal": "^3.1.3", + "fast-uri": "^3.0.1", + "json-schema-traverse": "^1.0.0", + "require-from-string": "^2.0.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, + "node_modules/ajv-formats/node_modules/json-schema-traverse": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", + "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==", + "license": "MIT" + }, "node_modules/amator": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/amator/-/amator-1.1.0.tgz", @@ -3952,7 +4067,6 @@ "version": "5.0.1", "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", - "dev": true, "engines": { "node": ">=8" } @@ -5484,8 +5598,7 @@ "node_modules/emoji-regex": { "version": "8.0.0", "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", - "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", - "dev": true + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==" }, "node_modules/encoding-down": { "version": "6.3.0", @@ -6042,8 +6155,7 @@ "node_modules/fast-deep-equal": { "version": "3.1.3", "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", - "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==", - "dev": true + "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==" }, "node_modules/fast-diff": { "version": "1.1.2", @@ -6102,7 +6214,6 @@ "version": "3.0.6", "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.0.6.tgz", "integrity": "sha512-Atfo14OibSv5wAp4VWNsFYE1AchQRTv9cBGWET4pZWHzYshFSS9NQI6I57rdKn9croWVMbYFbLhJ+yJvmZIIHw==", - "dev": true, "funding": [ { "type": "github", @@ -6112,8 +6223,7 @@ "type": "opencollective", "url": "https://opencollective.com/fastify" } - ], - "peer": true + ] }, "node_modules/fastest-levenshtein": { "version": "1.0.16", @@ -7178,7 +7288,6 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", - "dev": true, "engines": { "node": ">=8" } @@ -7360,6 +7469,15 @@ "resolved": "https://registry.npmjs.org/json-stringify-pretty-compact/-/json-stringify-pretty-compact-2.0.0.tgz", "integrity": "sha512-WRitRfs6BGq4q8gTgOy4ek7iPFXjbra0H3PmDLKm2xnZ+Gh1HUhiKGgCZkSPNULlP7mvfu6FV/mOLhCarspADQ==" }, + "node_modules/jsonpointer": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/jsonpointer/-/jsonpointer-5.0.1.tgz", + "integrity": "sha512-p/nXbhSEcu3pZRdkW1OfJhpsVtW1gd4Wa1fnQc9YLiTfAjn0312eMKimbdIQzuZl9aa9xUGaRlP9T/CJE/ditQ==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/keyv": { "version": "4.5.4", "resolved": "https://registry.npmjs.org/keyv/-/keyv-4.5.4.tgz", @@ -7539,6 +7657,18 @@ "node": ">=6" } }, + "node_modules/leven": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/leven/-/leven-4.0.0.tgz", + "integrity": "sha512-puehA3YKku3osqPlNuzGDUHq8WpwXupUg1V6NXdV38G+gr+gkBwFC8g1b/+YcIvp8gnqVIus+eJCH/eGsRmJNw==", + "license": "MIT", + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/levn": { "version": "0.4.1", "resolved": "https://registry.npmjs.org/levn/-/levn-0.4.1.tgz", @@ -9210,6 +9340,12 @@ "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz", "integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==" }, + "node_modules/openapi-types": { + "version": "12.1.3", + "resolved": "https://registry.npmjs.org/openapi-types/-/openapi-types-12.1.3.tgz", + "integrity": "sha512-N4YtSYJqghVu4iek2ZUvcN/0aqH1kRDuNqzcycDxhOUpg7GdvLa2F3DgS6yBNhInhv2r/6I0Flkn7CqL8+nIcw==", + "license": "MIT" + }, "node_modules/optionator": { "version": "0.9.4", "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.9.4.tgz", @@ -10713,8 +10849,6 @@ "version": "2.0.2", "resolved": "https://registry.npmjs.org/require-from-string/-/require-from-string-2.0.2.tgz", "integrity": "sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==", - "dev": true, - "peer": true, "engines": { "node": ">=0.10.0" } @@ -11162,7 +11296,6 @@ "version": "4.2.3", "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", - "dev": true, "dependencies": { "emoji-regex": "^8.0.0", "is-fullwidth-code-point": "^3.0.0", @@ -11191,7 +11324,6 @@ "version": "6.0.1", "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", - "dev": true, "dependencies": { "ansi-regex": "^5.0.1" }, @@ -13028,6 +13160,25 @@ "node": ">=0.4" } }, + "node_modules/xterm": { + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/xterm/-/xterm-5.3.0.tgz", + "integrity": "sha512-8QqjlekLUFTrU6x7xck1MsPzPA571K5zNqWm0M0oroYEWVOptZ0+ubQSkQ3uxIEhcIHRujJy6emDWX4A7qyFzg==", + "deprecated": "This package is now deprecated. Move to @xterm/xterm instead.", + "license": "MIT" + }, + "node_modules/xterm-readline": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/xterm-readline/-/xterm-readline-1.1.2.tgz", + "integrity": "sha512-1+W2nVuQvCYz9OUYwFBiolrSQUui51aDDyacKXt4PuxeBHqzvabQEJ2kwdBDzsmOjz5BwlDTAjJmYpH2OGqLFA==", + "license": "MIT", + "dependencies": { + "string-width": "^4" + }, + "peerDependencies": { + "@xterm/xterm": "^5.5.0" + } + }, "node_modules/y-leveldb": { "version": "0.1.2", "resolved": "https://registry.npmjs.org/y-leveldb/-/y-leveldb-0.1.2.tgz", @@ -13128,6 +13279,7 @@ "version": "2.8.0", "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.8.0.tgz", "integrity": "sha512-4lLa/EcQCB0cJkyts+FpIRx5G/llPxfP6VQU5KByHEhLxY3IJCH0f0Hy1MHI8sClTvsIb8qwRJ6R/ZdlDJ/leQ==", + "license": "ISC", "bin": { "yaml": "bin.mjs" }, @@ -13191,6 +13343,15 @@ "url": "https://github.com/sponsors/colinhacks" } }, + "node_modules/zod-to-json-schema": { + "version": "3.24.5", + "resolved": "https://registry.npmjs.org/zod-to-json-schema/-/zod-to-json-schema-3.24.5.tgz", + "integrity": "sha512-/AuWwMP+YqiPbsJx5D6TfgRTc4kTLjsh5SOcd4bLsfUg2RcEXrFMJl1DGgdHy2aCfsIA/cr/1JM0xcB2GZji8g==", + "license": "ISC", + "peerDependencies": { + "zod": "^3.24.1" + } + }, "node_modules/zstddec": { "version": "0.1.0", "resolved": "https://registry.npmjs.org/zstddec/-/zstddec-0.1.0.tgz", diff --git a/frontend/package.json b/frontend/package.json index e8a4179f75..b6ca38f2f5 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "windmill-components", - "version": "1.493.10", + "version": "1.496.3", "scripts": { "dev": "vite dev", "build": "vite build", @@ -80,18 +80,20 @@ "dependencies": { "@aws-crypto/sha256-js": "^4.0.0", "@codingame/monaco-vscode-configuration-service-override": "~16.1.1", + "@codingame/monaco-vscode-editor-api": "~16.1.1", "@codingame/monaco-vscode-standalone-css-language-features": "~16.1.1", "@codingame/monaco-vscode-standalone-html-language-features": "^16.1.1", "@codingame/monaco-vscode-standalone-json-language-features": "~16.1.1", "@codingame/monaco-vscode-standalone-languages": "~16.1.1", "@codingame/monaco-vscode-standalone-typescript-language-features": "~16.1.1", - "@codingame/monaco-vscode-editor-api": "~16.1.1", "@json2csv/plainjs": "^7.0.6", "@leeoniya/ufuzzy": "^1.0.8", "@popperjs/core": "^2.11.6", "@redocly/json-to-json-schema": "^0.0.1", + "@scalar/openapi-parser": "^0.15.0", "@tanstack/svelte-table": "npm:tanstack-table-8-svelte-5@^0.1", "@windmill-labs/svelte-dnd-action": "^0.9.48", + "@xterm/addon-fit": "^0.10.0", "@xyflow/svelte": "^0.1.15", "ag-charts-community": "^9.0.1", "ag-charts-enterprise": "^9.0.1", @@ -120,6 +122,7 @@ "monaco-vim": "^0.4.1", "ol": "^7.4.0", "openai": "^4.87.1", + "openapi-types": "^12.1.3", "p-limit": "^6.1.0", "panzoom": "^9.4.3", "pdfjs-dist": "4.8.69", @@ -147,11 +150,14 @@ "windmill-parser-wasm-ts": "^1.486.1", "windmill-parser-wasm-yaml": "^1.429.0", "windmill-sql-datatype-parser-wasm": "^1.318.0", + "xterm": "^5.3.0", + "xterm-readline": "^1.1.2", "y-monaco": "^0.1.4", "y-websocket": "^1.5.4", - "yaml": "^2.3.4", + "yaml": "^2.8.0", "yjs": "^13.6.7", - "zod": "^3.24.2" + "zod": "^3.24.2", + "zod-to-json-schema": "^3.24.5" }, "peerDependencies": { "svelte": "^5.0.0" diff --git a/frontend/src/lib/components/AppConnectInner.svelte b/frontend/src/lib/components/AppConnectInner.svelte index 6f73fff916..217d5981f3 100644 --- a/frontend/src/lib/components/AppConnectInner.svelte +++ b/frontend/src/lib/components/AppConnectInner.svelte @@ -63,7 +63,7 @@ (x) => passwords.includes(x) || ['token', 'secret', 'key', 'pass', 'private'].some((y) => x.toLowerCase().includes(y)) - ) + ) } let linkedSecret: string | undefined = undefined @@ -487,6 +487,8 @@ {#if !nativeLanguagesCategory.includes(key)} + + {:else} + + {/if} + + + + + + + + + +
+ +
diff --git a/frontend/src/lib/components/HttpAgentWorkerDrawer.svelte b/frontend/src/lib/components/HttpAgentWorkerDrawer.svelte index 7250a906c8..9a505bad55 100644 --- a/frontend/src/lib/components/HttpAgentWorkerDrawer.svelte +++ b/frontend/src/lib/components/HttpAgentWorkerDrawer.svelte @@ -8,16 +8,15 @@ import { enterpriseLicense, superadmin } from '$lib/stores' import CollapseLink from './CollapseLink.svelte' - export let customTags: string[] | undefined - let selectedTags: string[] = !$enterpriseLicense ? ['agent_test'] : [] + type Props = { + customTags: string[] | undefined + } + let { customTags = $bindable() }: Props = $props() + let selectedTags: string[] = $state(!$enterpriseLicense ? ['agent_test'] : []) + let workerGroup: string = $state('agent') + let token: string = $state('') - let workerGroup: string = 'agent' - - let token: string = '' - - $: selectedTags && selectedTags.length > 0 && $superadmin && workerGroup && refreshToken() - - async function refreshToken() { + async function refreshToken(workerGroup: string, selectedTags: string[]) { try { const newToken = await AgentWorkersService.createAgentToken({ requestBody: { @@ -32,6 +31,12 @@ sendUserToast('Error creating agent token: ' + error.toString(), true) } } + + $effect(() => { + if (selectedTags.length > 0 && $superadmin) { + refreshToken(workerGroup, selectedTags) + } + })
@@ -55,64 +60,81 @@ {/if} +
-
+ {#if !$enterpriseLicense} +
+ Agent workers are only available in the enterprise edition. For evaluation purposes, you can + only use the tag `agent_test` tag and it is limited to 100 jobs. +
+ {/if} +
+ { - navigator.clipboard.writeText(token) - sendUserToast('Copied to clipboard') + on:click|preventDefault|stopPropagation={() => { + if (token) { + navigator.clipboard.writeText(token) + sendUserToast('Copied to clipboard') + } }} - placeholder="Select tags to generate a jwt token" + placeholder="Select tags to generate a JWT token" type="text" disabled value={token} - class="pr-8 text-sm text-secondary" + class="w-full pr-10 pl-3 py-2 text-sm text-gray-600 bg-gray-50 border border-gray-300 rounded-lg cursor-pointer hover:bg-gray-100 transition truncate" /> + +
-
- Pass the env variables: -
    -
  • MODE=agent
  • -
  • AGENT_TOKEN={'""'}
  • -
  • BASE_INTERNAL_URL={'""'}
  • +
    + Set the following environment variables: +
      +
    • MODE=agent
    • +
    • AGENT_TOKEN=<token>
    • +
    • BASE_INTERNAL_URL=<base url>
    - to a worker to have it act as an HTTP agent worker. INIT_SCRIPT, if needed, must be passed as an - env variable. - -

    - Remember to have at least one normal worker that listens to the tags `flow` and `dependency` - (or `flow-$workspace` and `dependency-$workspace` if using workspace specific default tags) - to have flow and dependency job being runnable as agent workers can't run dependency jobs - nor can run the flow state machine (but can run the subjobs within them). +

    + to a worker to have it act as an HTTP agent worker. + INIT_SCRIPT, if needed, must be passed as an env variable.

    + + Ensure at least one normal worker is running and listening to the tags + flow and dependency + (or flow-<workspace> and dependency-<workspace> if + using workspace-specific default tags), because agent workers + cannot run dependency jobs + nor execute the + flow state machine. They can, however, run subjobs within flows. + + +
    + Use the following API endpoint with a superadmin bearer token: + POST /api/agent_workers/create_agent_token +
    +	{`
    +	  "worker_group": "agent",
    +	  "tags": ["tag1", "tag2"],
    +	  "exp": 1717334400
    +	`}
    +					
    + The JSON response will contain the generated JWT token. +
    +
    - - -
    - Use the following api endpoint with a superadmin bearer token: - POST /api/agent_workers/create_agent_token - with body: -
    -			
    -{`{
    -	"worker_group": "agent",
    -	"tags": ["tag1", "tag2"],
    -	"exp": 1717334400 // 3 years from now
    -}`}
    -					
    -				
    - JSON response will be the JWT token. -
    -
diff --git a/frontend/src/lib/components/InstanceSetting.svelte b/frontend/src/lib/components/InstanceSetting.svelte index 6e75df9270..76ee0863b8 100644 --- a/frontend/src/lib/components/InstanceSetting.svelte +++ b/frontend/src/lib/components/InstanceSetting.svelte @@ -165,8 +165,12 @@ } } - function handleTeamChange(teamItem: { team_id: string; team_name: string }, i: number) { - const team = $values['teams'].find((team) => team.team_id === teamItem.team_id) || null + function handleTeamChange( + teamItem: { team_id: string; team_name: string } | undefined, + i: number + ) { + const team = + (teamItem && $values['teams'].find((team) => team.team_id === teamItem.team_id)) || null $values['critical_error_channels'][i] = { teams_channel: { team_id: team?.team_id, @@ -177,7 +181,10 @@ } } - function handleChannelChange(channel: { channel_id: string; channel_name: string }, i: number) { + function handleChannelChange( + channel: { channel_id: string; channel_name: string } | undefined, + i: number + ) { const team = $values['critical_error_channels'][i]?.teams_channel if (team) { $values['critical_error_channels'][i] = { @@ -550,15 +557,19 @@ showRefreshButton={false} placeholder="Select team" teams={$values['teams']} - on:change={(e) => handleTeamChange(e.detail, i)} - selectedTeam={$values['critical_error_channels'][i]?.teams_channel - ? { - team_id: - $values['critical_error_channels'][i]?.teams_channel?.team_id, - team_name: - $values['critical_error_channels'][i]?.teams_channel?.team_name - } - : undefined} + bind:selectedTeam={ + () => + $values['critical_error_channels'][i]?.teams_channel + ? { + team_id: + $values['critical_error_channels'][i]?.teams_channel?.team_id, + team_name: + $values['critical_error_channels'][i]?.teams_channel + ?.team_name + } + : undefined, + (team) => handleTeamChange(team, i) + } /> {#if $values['critical_error_channels'][i]?.teams_channel?.team_id} @@ -570,18 +581,20 @@ team.team_id === $values['critical_error_channels'][i]?.teams_channel?.team_id )?.channels ?? []} - on:change={(e) => handleChannelChange(e.detail, i)} - selectedChannel={$values['critical_error_channels'][i]?.teams_channel - ?.channel_id - ? { - channel_id: - $values['critical_error_channels'][i]?.teams_channel - ?.channel_id, - channel_name: - $values['critical_error_channels'][i]?.teams_channel - ?.channel_name - } - : undefined} + bind:selectedChannel={ + () => + $values['critical_error_channels'][i]?.teams_channel?.channel_id + ? { + channel_id: + $values['critical_error_channels'][i]?.teams_channel + ?.channel_id, + channel_name: + $values['critical_error_channels'][i]?.teams_channel + ?.channel_name + } + : undefined, + (channel) => handleChannelChange(channel, i) + } /> {/if}
diff --git a/frontend/src/lib/components/LightweightResourcePicker.svelte b/frontend/src/lib/components/LightweightResourcePicker.svelte index 8f8e77c158..50da38607b 100644 --- a/frontend/src/lib/components/LightweightResourcePicker.svelte +++ b/frontend/src/lib/components/LightweightResourcePicker.svelte @@ -2,15 +2,13 @@ import { ResourceService } from '$lib/gen' import { workspaceStore } from '$lib/stores' import { createEventDispatcher, getContext } from 'svelte' - import Select from './apps/svelte-select/lib/index' - import { SELECT_INPUT_DEFAULT_STYLE } from '../defaults' - import DarkModeObserver from './DarkModeObserver.svelte' import { Button, Drawer, DrawerContent } from './common' import { Plus, Loader2, Link2Off } from 'lucide-svelte' import type { AppViewerContext } from './apps/types' import { sendUserToast } from '$lib/toast' import { createDispatcherIfMounted } from '$lib/createDispatcherIfMounted' + import Select from './Select.svelte' const dispatch = createEventDispatcher() const dispatchIfMounted = createDispatcherIfMounted(dispatch) @@ -34,7 +32,7 @@ } : undefined - let collection = [valueSelect] + let collection = valueSelect ? [valueSelect] : [] let loading = true async function loadResources(resourceType: string | undefined) { @@ -167,23 +165,22 @@ {:else} - -
- {:else} - - {/if} - - - - - - - - -
{#if meta != undefined} @@ -474,44 +422,7 @@ {:else if meta.ownerKind === 'folder'} {/if}
diff --git a/frontend/src/lib/components/ResourcePicker.svelte b/frontend/src/lib/components/ResourcePicker.svelte index 4e50c13911..6169fc10e7 100644 --- a/frontend/src/lib/components/ResourcePicker.svelte +++ b/frontend/src/lib/components/ResourcePicker.svelte @@ -2,18 +2,16 @@ import { ResourceService } from '$lib/gen' import { workspaceStore } from '$lib/stores' import { createEventDispatcher, onMount } from 'svelte' - import Select from './apps/svelte-select/lib/index' - import { SELECT_INPUT_DEFAULT_STYLE } from '../defaults' import AppConnect from './AppConnectDrawer.svelte' import ResourceEditorDrawer from './ResourceEditorDrawer.svelte' import { Button } from './common' import DBManagerDrawerButton from './DBManagerDrawerButton.svelte' - import DarkModeObserver from './DarkModeObserver.svelte' import { Pen, Plus, RotateCw } from 'lucide-svelte' import { sendUserToast } from '$lib/toast' import { isDbType } from './apps/components/display/dbtable/utils' import { createDispatcherIfMounted } from '$lib/createDispatcherIfMounted' + import Select from './Select.svelte' const dispatch = createEventDispatcher() const dispatchIfMounted = createDispatcherIfMounted(dispatch) @@ -104,12 +102,8 @@ let appConnect: AppConnect let resourceEditor: ResourceEditorDrawer - - let darkMode: boolean = false - - { await loadResources(resourceType) @@ -141,14 +135,16 @@ {#if collection?.length > 0} {:else} filterText, (v) => (filterText = v)} + placeholder={loading ? 'Loading...' : (valueEntry?.label ?? placeholder)} + style={containerStyle} + class={twMerge( + '!bg-surface text-ellipsis', + open ? '' : 'cursor-pointer', + valueEntry && !loading ? '!placeholder-primary' : '', + clearable && !disabled && value ? '!pr-8' : '', + inputClass ?? '' + )} + autocomplete="off" + onpointerdown={() => (open = true)} + bind:this={inputEl} + /> + + + {#if open && !disabled} +
+ {#if processedItems?.length === 0} +
No items
+ {/if} + {#each processedItems ?? [] as item, itemIndex} + {#if (item.__select_group && itemIndex === 0) || processedItems?.[itemIndex - 1]?.__select_group !== item.__select_group} +
+ {item.__select_group} +
+ {/if} + + {/each} +
+ {/if} +
+
diff --git a/frontend/src/lib/components/ServiceLogsInner.svelte b/frontend/src/lib/components/ServiceLogsInner.svelte index 9b358adf80..6866c3e7a7 100644 --- a/frontend/src/lib/components/ServiceLogsInner.svelte +++ b/frontend/src/lib/components/ServiceLogsInner.svelte @@ -15,9 +15,6 @@ import AnsiUp from 'ansi_up' import { scroll_into_view_if_needed_polyfill } from './multiselect/utils' import SplitPanesOrColumnOnMobile from './splitPanes/SplitPanesOrColumnOnMobile.svelte' - import Select from './apps/svelte-select/lib/Select.svelte' - import { SELECT_INPUT_DEFAULT_STYLE } from '$lib/defaults' - import DarkModeObserver from './DarkModeObserver.svelte' export let searchTerm: string export let queryParseErrors: string[] = [] @@ -398,23 +395,8 @@ return ret } - - function getSelectItems(allLogs: ByMode, countsPerHost: any): { label: string; value: any }[] { - return Object.entries(allLogsOrQueryResults(allLogs, countsPerHost)).flatMap(([mode, o1]) => - Object.entries(o1).flatMap(([wg, o2]) => - Object.keys(o2).map((hn) => ({ - label: hn, - value: [mode, wg, hn] - })) - ) - ) - } - - let darkMode = false - - @@ -569,24 +551,6 @@ > {/if} -
- ({ + label: team.team_name, + value: team.team_id + })) ?? []} {placeholder} - searchable={true} - loading={isFetching} + clearable disabled={disabled || isFetching} - on:input={(e) => filterTeams(e.detail)} + bind:value={ + () => selectedTeam?.team_id, + (value) => (selectedTeam = teams?.find((team) => team.team_id === value)) + } />
{#if showRefreshButton} + {:else} +
+ +
+ +
+ + +
+
+
+ + +
+
+ +
+ + + diff --git a/frontend/src/lib/components/WorkerTagSelect.svelte b/frontend/src/lib/components/WorkerTagSelect.svelte index 8cac3f0a64..616cb6d809 100644 --- a/frontend/src/lib/components/WorkerTagSelect.svelte +++ b/frontend/src/lib/components/WorkerTagSelect.svelte @@ -2,10 +2,8 @@ import { workerTags, workspaceStore } from '$lib/stores' import { WorkerService } from '$lib/gen' - import Select from './apps/svelte-select/lib/index' import { createEventDispatcher } from 'svelte' - import DarkModeObserver from './DarkModeObserver.svelte' - import { SELECT_INPUT_DEFAULT_STYLE } from '$lib/defaults' + import Select from './Select.svelte' let { tag = $bindable(), @@ -32,57 +30,18 @@ ...(tag && tag != '' && !($workerTags ?? []).includes(tag) ? [tag] : []), ...($workerTags ?? []) ]) - let darkMode: boolean = $state(false) - -
{#if !noLabel}
tag
{/if} { - selectedDay = e.detail.value - outputs.day.set(Number(selectedDay)) - }} - on:clear={() => { - selectedDay = '' - outputs.day.set(undefined) - }} + bind:value={ + () => selectedDay, + (v) => { + selectedDay = v ?? '' + outputs.day.set(v ? Number(v) : undefined) + } + } items={Array.from({ length: computeDayPerMonth(selectedMonth, selectedYear) }, (_, i) => { return { label: String(i + 1), value: String(i + 1) } })} class={twMerge('text-clip min-w-0', css?.input?.class, 'wm-date-select')} - containerStyles={(darkMode - ? SELECT_INPUT_DEFAULT_STYLE.containerStylesDark - : SELECT_INPUT_DEFAULT_STYLE.containerStyles) + css?.input?.style} - inputStyles={SELECT_INPUT_DEFAULT_STYLE.inputStyles} + containerStyle={css?.input?.style} placeholder="Pick a day" + clearable />
{/if} @@ -263,22 +253,17 @@ class={twMerge('grow', resolvedConfig?.orientation === 'horizontal' ? 'w-1/2' : 'w-full')} > { - selectedYear = e.detail.value - outputs.year.set(Number(selectedYear)) - }} - on:clear={() => { - selectedYear = '' - outputs.year.set(undefined) - }} - items={Array.from({ length: 201 }, (_, i) => `${1900 + i}`)} + bind:value={ + () => selectedYear, + (v) => { + selectedYear = v ?? '' + outputs.year.set(selectedYear ? Number(selectedYear) : undefined) + } + } + items={Array.from({ length: 201 }, (_, i) => `${1900 + i}`).map((value) => ({ + value + }))} placeholder="Pick a year" - inputStyles={SELECT_INPUT_DEFAULT_STYLE.inputStyles} class={twMerge('text-clip min-w-0', css?.input?.class, 'wm-date-select')} - containerStyles={(darkMode - ? SELECT_INPUT_DEFAULT_STYLE.containerStylesDark - : SELECT_INPUT_DEFAULT_STYLE.containerStyles) + css?.input?.style} + containerStyle={css?.input?.style} clearable /> diff --git a/frontend/src/lib/components/apps/components/inputs/AppSelect.svelte b/frontend/src/lib/components/apps/components/inputs/AppSelect.svelte index 6d7abcaf55..2f35f531dd 100644 --- a/frontend/src/lib/components/apps/components/inputs/AppSelect.svelte +++ b/frontend/src/lib/components/apps/components/inputs/AppSelect.svelte @@ -1,6 +1,5 @@ - - -{#if selectOptions && componentInput} +{#if options && componentInput} { - if (copilotModule.description.length > 2) { - getHubCompletions(copilotModule.description, i, copilotModule.type) - } else { - copilotModule.hubCompletions = [] - } - }} - /> - {/if} - {#if copilotModule.description.length > 2 && copilotModule.source === undefined} -
- - -
- {#if copilotModule.hubCompletions.length > 0} -

Hub scripts

-
    - {#each copilotModule.hubCompletions as item (item.path)} -
  • - -
  • - {/each} -
- {/if} - {/if} - - - {/each} - {#if flowCopilotMode !== 'trigger'} -
- -
- {/if} - - - -
-
diff --git a/frontend/src/lib/components/copilot/FlowCopilotStatus.svelte b/frontend/src/lib/components/copilot/FlowCopilotStatus.svelte deleted file mode 100644 index d4d2a781fb..0000000000 --- a/frontend/src/lib/components/copilot/FlowCopilotStatus.svelte +++ /dev/null @@ -1,145 +0,0 @@ - - -{#if $copilotInfo.enabled} - - -
- - {copilotStatus} - - {#if !copilotLoading && $currentStepStore !== undefined && $currentStepStore !== 'Input'} - - - {/if} -
-
-{:else} - - - - - -
-

Enable Windmill AI in the workspace settings

-
-
-
-{/if} diff --git a/frontend/src/lib/components/copilot/MetadataGen.svelte b/frontend/src/lib/components/copilot/MetadataGen.svelte index 3eaa1d20e3..9c42bafcdf 100644 --- a/frontend/src/lib/components/copilot/MetadataGen.svelte +++ b/frontend/src/lib/components/copilot/MetadataGen.svelte @@ -11,6 +11,7 @@ import { yamlStringifyExceptKeys } from './utils' import type { ChatCompletionMessageParam } from 'openai/resources/index.mjs' import { createDispatcherIfMounted } from '$lib/createDispatcherIfMounted' + import TriggerableByAI from '$lib/components/TriggerableByAI.svelte' type PromptConfig = { system: string @@ -78,6 +79,8 @@ Generate a description for the flow below: } } + export let aiId: string | undefined = undefined + export let aiDescription: string | undefined = undefined export let content: string | undefined export let code: string | undefined = undefined export let flow: FlowValue | undefined = undefined @@ -187,104 +190,117 @@ Generate a description for the flow below: -
{ - if (!$copilotInfo.enabled || !$metadataCompletionEnabled) { - return - } - if (event.key === 'Tab') { - if (manualDisabled) { - event.preventDefault() - manualDisabled = false - } else if (!loading && generatedContent) { - event.preventDefault() - content = generatedContent - generatedContent = '' - } else if (!loading && !content) { - event.preventDefault() - generateContent() - } - } else if (event.key === 'Escape' && !manualDisabled) { - event.preventDefault() - event.stopPropagation() - if (loading) { - abortController.abort() - } else { - manualDisabled = true - generatedContent = '' - } - } else if (event.key === 'Backspace' && !loading && !content) { - manualDisabled = true - generatedContent = '' + { + if (value) { + content = value + dispatchIfMounted('change', { content }) } }} >
{ + if (!$copilotInfo.enabled || !$metadataCompletionEnabled) { + return + } + if (event.key === 'Tab') { + if (manualDisabled) { + event.preventDefault() + manualDisabled = false + } else if (!loading && generatedContent) { + event.preventDefault() + content = generatedContent + generatedContent = '' + } else if (!loading && !content) { + event.preventDefault() + generateContent() + } + } else if (event.key === 'Escape' && !manualDisabled) { + event.preventDefault() + event.stopPropagation() + if (loading) { + abortController.abort() + } else { + manualDisabled = true + generatedContent = '' + } + } else if (event.key === 'Backspace' && !loading && !content) { + manualDisabled = true + generatedContent = '' + } + }} > - {#if active} - 0 - ? 'bg-green-100 text-green-800 dark:text-green-400 dark:bg-green-700' - : '' - )} - > - - {#if loading} - ESC - {:else} - TAB - {/if} - {#if loading} - - {:else if generatedContent} - - {:else} - - {/if} +
+ {#if active} + 0 + ? 'bg-green-100 text-green-800 dark:text-green-400 dark:bg-green-700' + : '' + )} + > + + {#if loading} + ESC + {:else} + TAB + {/if} + {#if loading} + + {:else if generatedContent} + + {:else} + + {/if} + - -
- {generatedContent} +
+ {generatedContent} +
+ {/if} +
+ {#if elementType === 'textarea'} +
+
GH Markdown
+
- {/if} -
- {#if elementType === 'textarea'} -
-
GH Markdown
- -
- {:else} - (focused = true)} - on:blur={() => (focused = false)} - /> - {/if} - -
+ /> + {/if} + +
+ diff --git a/frontend/src/lib/components/copilot/ScriptFix.svelte b/frontend/src/lib/components/copilot/ScriptFix.svelte index a6dd7e2cdb..568abeb507 100644 --- a/frontend/src/lib/components/copilot/ScriptFix.svelte +++ b/frontend/src/lib/components/copilot/ScriptFix.svelte @@ -2,172 +2,23 @@ import { base } from '$lib/base' import { Button } from '../common' - import { SUPPORTED_LANGUAGES, copilot } from './lib' + import { SUPPORTED_LANGUAGES } from './lib' import type { SupportedLanguage } from '$lib/common' - import { sendUserToast } from '$lib/toast' - import type Editor from '../Editor.svelte' - import { dbSchemas, copilotInfo, type DBSchema, workspaceStore } from '$lib/stores' - import type DiffEditor from '../DiffEditor.svelte' - import { scriptLangToEditorLang } from '$lib/scripts' import Popover from '$lib/components/meltComponents/Popover.svelte' - import { writable } from 'svelte/store' - import { WindmillIcon } from '../icons' - import HighlightCode from '../HighlightCode.svelte' - import LoadingIcon from '../apps/svelte-select/lib/LoadingIcon.svelte' import { autoPlacement } from '@floating-ui/core' - import { Check, Wand2, X, RotateCw } from 'lucide-svelte' - import { createEventDispatcher } from 'svelte' + import { WandSparkles } from 'lucide-svelte' + import { aiChatManager } from './chat/AIChatManager.svelte' + import { copilotInfo } from '$lib/stores' - // props - export let lang: SupportedLanguage - export let editor: Editor | undefined - export let diffEditor: DiffEditor | undefined - export let error: string - export let args: Record - export let chatMode: boolean = false - - // state - let genLoading: boolean = false - let generatedCode = writable('') - let generatedExplanation = writable('') - let dbSchema: DBSchema | undefined = undefined - let abortController: AbortController | undefined = undefined - - const dispatch = createEventDispatcher<{ - fix: null - }>() - - async function onFix() { - if (!error) { - return - } - try { - genLoading = true - abortController = new AbortController() - await copilot( - { - language: lang, - code: editor?.getCode() || '', - error, - dbSchema: dbSchema, - type: 'fix', - workspace: $workspaceStore! - }, - generatedCode, - abortController, - generatedExplanation - ) - setupDiff() - diffEditor?.setModified($generatedCode) - showDiff() - } catch (err) { - if (!abortController?.signal.aborted) { - if (err?.message) { - sendUserToast('Failed to generate code: ' + err.message, true) - } else { - sendUserToast('Failed to generate code', true) - console.error(err) - } - } - } finally { - genLoading = false - } - } - - function acceptDiff() { - editor?.setCode(diffEditor?.getModified() || '') - editor?.format() - clear() - } - - function rejectDiff() { - clear() - } - - function setupDiff() { - diffEditor?.setupModel(scriptLangToEditorLang(lang)) - diffEditor?.setOriginal(editor?.getCode() || '') - } - - function showDiff() { - diffEditor?.show() - editor?.hide() - } - - function hideDiff() { - editor?.show() - diffEditor?.hide() - } - - function clear() { - $generatedCode = '' - $generatedExplanation = '' - } - - $: lang && clear() - - $: !$generatedCode && hideDiff() - - function updateSchema(lang, args) { - const schemaRes = lang === 'graphql' ? args.api : args.database - if (typeof schemaRes === 'string') { - dbSchema = $dbSchemas[schemaRes.replace('$res:', '')] - } - } - $: updateSchema(lang, args) - - let popover: Popover | undefined = undefined + let { + lang + }: { + lang: SupportedLanguage + } = $props() {#if SUPPORTED_LANGUAGES.has(lang)} - {#if !genLoading && $generatedCode.length > 0} -
- - -
- {/if} - +
- {#if $copilotInfo.enabled} -
- {#if $generatedCode.length > 0 && genLoading} -
- -
- {:else if genLoading} - - {/if} - {#if $generatedExplanation.length > 0} -

Explanation: test {$generatedExplanation}

- {/if} -
- {:else} -
-

Enable Windmill AI in the workspace settings

- {/if} +
+

Enable Windmill AI in the workspace settings

diff --git a/frontend/src/lib/components/copilot/StepGen.svelte b/frontend/src/lib/components/copilot/StepGen.svelte deleted file mode 100644 index cc8fb5efde..0000000000 --- a/frontend/src/lib/components/copilot/StepGen.svelte +++ /dev/null @@ -1,246 +0,0 @@ - - - (emptyString(x.summary) ? x.path : x.summary + ' (' + x.path + ')')} -/> - -
-
-
- { - if (funcDesc?.length > 2) { - getHubCompletions(funcDesc) - } else { - hubCompletions = [] - } - }} - placeholder="Search {trigger ? 'triggers' : 'scripts'} or AI gen" - /> - {#if funcDesc?.length === 0} - - {/if} -
- {#if !disableAi && funcDesc?.length > 0} -
    -
  • - -
  • -
  • - -
  • -
- {/if} - {#if funcDesc?.length > 0 && filteredItems?.length > 0} -
-

Workspace {trigger ? 'Triggers' : 'Scripts'}

-
    - {#each filteredItems?.slice(0, 3) ?? [] as item (item.path)} -
  • - -
  • - {/each} -
-
- {/if} - {#if hubCompletions.length > 0} -
-

Hub {trigger ? 'Triggers' : 'Scripts'}

-
    - {#each hubCompletions as item (item.path)} -
  • - -
  • - {/each} -
-
- {/if} -
-
diff --git a/frontend/src/lib/components/copilot/autocomplete/request.ts b/frontend/src/lib/components/copilot/autocomplete/request.ts index 0d6700a2b4..9c96f7610b 100644 --- a/frontend/src/lib/components/copilot/autocomplete/request.ts +++ b/frontend/src/lib/components/copilot/autocomplete/request.ts @@ -2,7 +2,7 @@ import { codeCompletionLoading, copilotInfo } from '$lib/stores' import { get } from 'svelte/store' import { getNonStreamingCompletion } from '../lib' -import { getLangContext } from '../chat/core' +import { getLangContext } from '../chat/script/core' import { type ScriptLang } from '$lib/gen/types.gen' const AUTOCOMPLETE_SYSTEM_PROMPT = `You're a code assistant. Your task is to help the user write code by suggesting the next edit for the user. diff --git a/frontend/src/lib/components/copilot/chat/AIChat.svelte b/frontend/src/lib/components/copilot/chat/AIChat.svelte index fa83e72346..d664c6aa05 100644 --- a/frontend/src/lib/components/copilot/chat/AIChat.svelte +++ b/frontend/src/lib/components/copilot/chat/AIChat.svelte @@ -1,565 +1,149 @@ + { + if ((e.ctrlKey || e.metaKey) && e.key === 'l') { + e.preventDefault() + aiChatManager.toggleOpen() + aiChatDisplay?.focusInput() + } + }} +/> + +{#snippet headerLeft()} +