diff --git a/.github/DockerfileBackendTests b/.github/DockerfileBackendTests index 87a65214ab..2ce47ea714 100644 --- a/.github/DockerfileBackendTests +++ b/.github/DockerfileBackendTests @@ -28,7 +28,7 @@ ENV PATH="${PATH}:/usr/local/go/bin" ENV GO_PATH=/usr/local/go/bin/go # UV -RUN curl --proto '=https' --tlsv1.2 -LsSf https://github.com/astral-sh/uv/releases/download/0.11.24/uv-installer.sh | sh && mv /usr/local/cargo/bin/uv /usr/local/bin/uv +RUN curl --proto '=https' --tlsv1.2 -LsSf https://github.com/astral-sh/uv/releases/download/0.13.0/uv-installer.sh | sh && mv /usr/local/cargo/bin/uv /usr/local/bin/uv ENV TZ=Etc/UTC diff --git a/.github/workflows/ai-evals-scheduled.yml b/.github/workflows/ai-evals-scheduled.yml index cc1c0d6763..580a757c21 100644 --- a/.github/workflows/ai-evals-scheduled.yml +++ b/.github/workflows/ai-evals-scheduled.yml @@ -1,25 +1,22 @@ name: AI Evals (scheduled) -# Full ai_evals suites on current models, posted to the AI evals dashboard in -# the windmill-prod workspace (f/ai/ai_evals_dashboard) so quality is tracked -# over time. Weekly, since one pass of every suite at --runs 3 costs about 50M -# tokens per model: every suite runs on MODELS, and global (the mode users get) -# also runs on GLOBAL_EXTRA_MODELS, one flagship per other provider. Run it by -# hand to measure a branch against main. +# The global (AI chat) and cli suites on current models, posted to the AI evals +# dashboard in the windmill-prod workspace (f/ai/ai_evals_dashboard) so quality is +# tracked over time. Weekly: one global pass at --runs 3 is about 45M tokens per +# model. Each mode and model is its own job, so a slow model neither delays nor +# shares a timeout with the others. Run it by hand to measure a branch against +# main, or to include the flow, app and script suites. on: schedule: - cron: "0 3 * * 1" workflow_dispatch: inputs: modes: - description: "Space-separated modes" - default: "global flow app script cli" + description: "Space-separated modes (global, cli, flow, app, script)" + default: "global cli" models: - description: "Space-separated model aliases (bun run cli -- models)" - default: "sonnet-5.5" - global_extra_models: - description: "Extra model aliases for global mode only" - default: "gpt-6-astra gemini-3.8-flash" + description: "Space-separated model aliases (bun run cli -- models); cli skips those it has no agent for" + default: "sonnet-5.5 opus-5.5 gpt-6.1-sol gpt-6-astra deepseek-v4-pro" runs: description: "Runs per case" default: "3" @@ -31,8 +28,7 @@ concurrency: group: ai-evals-scheduled-${{ github.ref }} env: - MODELS: ${{ inputs.models || 'sonnet-5.5' }} - GLOBAL_EXTRA_MODELS: ${{ inputs.global_extra_models || 'gpt-6-astra gemini-3.8-flash' }} + MODELS: ${{ inputs.models || 'sonnet-5.5 opus-5.5 gpt-6.1-sol gpt-6-astra deepseek-v4-pro' }} RUNS: ${{ inputs.runs || '3' }} REASONING: ${{ inputs.reasoning }} # Routes on the cloud instance are prefixed with their workspace. @@ -42,14 +38,31 @@ jobs: setup: runs-on: ubuntu-latest outputs: - modes: ${{ steps.modes.outputs.modes }} + matrix: ${{ steps.matrix.outputs.matrix }} steps: - - id: modes + - uses: actions/checkout@v4 + with: + sparse-checkout: ai_evals/core + + - uses: oven-sh/setup-bun@v2 + with: + bun-version: 1.4.0 + + - id: matrix env: - MODES: ${{ inputs.modes || 'global flow app script cli' }} + MODES: ${{ inputs.modes || 'global cli' }} + # cli takes only the aliases with a `cli` entry in ai_evals/core/models.ts. run: | - echo "modes=$(jq -cn --arg m "$MODES" '$m | split(" ") - | map(select(IN("global", "flow", "app", "script", "cli")))')" >> "$GITHUB_OUTPUT" + cli_models=$(bun -e ' + import { EVAL_MODELS } from "./ai_evals/core/models.ts"; + console.log(JSON.stringify(EVAL_MODELS.filter((m) => m.cli) + .flatMap((m) => [m.id, ...m.aliases]).map((a) => a.toLowerCase())));') + echo "matrix=$(jq -cn --arg modes "$MODES" --arg models "$MODELS" --argjson cli "$cli_models" ' + def words: split(" ") | map(select(. != "")); + [ $modes | words[] | select(IN("global", "flow", "app", "script", "cli")) as $mode + | $models | words[] + | select($mode != "cli" or (ascii_downcase | IN($cli[]))) + | {mode: $mode, model: .} ]')" >> "$GITHUB_OUTPUT" evals: needs: setup @@ -58,7 +71,7 @@ jobs: strategy: fail-fast: false matrix: - mode: ${{ fromJSON(needs.setup.outputs.modes) }} + include: ${{ fromJSON(needs.setup.outputs.matrix) }} services: postgres: image: postgres:16 @@ -118,10 +131,15 @@ jobs: working-directory: ./cli run: bun install && ./gen_wm_client.sh && ./windmill-utils-internal/gen_wm_client.sh - - name: Run ${{ matrix.mode }} evals + - name: Install Codex CLI + if: matrix.mode == 'cli' + run: npm install --global @openai/codex@0.159.3 + + - name: Run ${{ matrix.mode }} evals on ${{ matrix.model }} working-directory: ./ai_evals env: MODE: ${{ matrix.mode }} + MODEL: ${{ matrix.model }} WMILL_AI_EVAL_BACKEND_URL: http://localhost:8000 WMILL_AI_EVAL_BACKEND_WORKSPACE: integration-tests ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} @@ -131,14 +149,10 @@ jobs: run: | bun install mkdir -p results - models="$MODELS" - [ "$MODE" = global ] && models="$models $GLOBAL_EXTRA_MODELS" reasoning=() [ -n "$REASONING" ] && [ "$MODE" != cli ] && reasoning=(--reasoning "$REASONING") - for m in $models; do - bun run cli -- run "$MODE" --model "$m" --runs "$RUNS" "${reasoning[@]}" \ - --output "$PWD/results/$MODE-$m.json" || echo "::warning::$MODE on $m errored" - done + bun run cli -- run "$MODE" --model "$MODEL" --runs "$RUNS" "${reasoning[@]}" \ + --output "$PWD/results/$MODE-$MODEL.json" || echo "::warning::$MODE on $MODEL errored" - name: Post results to the dashboard if: always() @@ -166,7 +180,7 @@ jobs: uses: actions/upload-artifact@v4 if: always() with: - name: ai-evals-${{ matrix.mode }} + name: ai-evals-${{ matrix.mode }}-${{ matrix.model }} path: | ai_evals/logs ai_evals/results diff --git a/.github/workflows/backend-test-windows.yml b/.github/workflows/backend-test-windows.yml index 54a62ec495..3468449753 100644 --- a/.github/workflows/backend-test-windows.yml +++ b/.github/workflows/backend-test-windows.yml @@ -63,7 +63,7 @@ jobs: - uses: denoland/setup-deno@v2 with: - # Pin to the Deno version shipped in the runtime image (Dockerfile) so CI + # Pin to the Deno version shipped in the full runtime image (docker/DockerfileFull) so CI # tests what production runs, instead of floating on the latest v2.x. deno-version: 2.2.1 @@ -81,7 +81,7 @@ jobs: - uses: astral-sh/setup-uv@v7 with: - version: "0.11.24" + version: "0.13.0" - uses: shivammathur/setup-php@v2 with: diff --git a/.github/workflows/backend-test.yml b/.github/workflows/backend-test.yml index a2087025b7..73f955131e 100644 --- a/.github/workflows/backend-test.yml +++ b/.github/workflows/backend-test.yml @@ -50,7 +50,7 @@ jobs: dotnet-version: "9.0.x" - uses: denoland/setup-deno@v2 with: - # Pin to the Deno version shipped in the runtime image (Dockerfile) so CI + # Pin to the Deno version shipped in the full runtime image (docker/DockerfileFull) so CI # tests what production runs, instead of floating on the latest v2.x. deno-version: 2.2.1 - uses: actions/setup-go@v7 @@ -64,7 +64,7 @@ jobs: node-version: "24" - uses: astral-sh/setup-uv@v7 with: - version: "0.11.24" + version: "0.13.0" - uses: shivammathur/setup-php@v2 with: php-version: "8.3" @@ -295,4 +295,4 @@ jobs: # never reaches it. Pin the target dir (matching the cache step above) so # its own tests run off this compile rather than a second bundled build. (cd windmill-duckdb-ffi-internal && export CARGO_TARGET_DIR="$PWD/target" && ./build_dev.sh && cargo test --release -p windmill_duckdb_ffi_internal) - DENO_PATH=$(which deno) BUN_PATH=$(which bun) NODE_BIN_PATH=$(which node) GO_PATH=$(which go) UV_PATH=$(which uv) PHP_PATH=$(which php) COMPOSER_PATH=$(which composer) RUBY_PATH=$(which ruby) RUBY_BUNDLE_PATH=$(which bundle) RUBY_GEM_PATH=$(which gem) POWERSHELL_PATH=$(which pwsh) DOTNET_PATH=$(which dotnet) cargo test --features enterprise,deno_core,duckdb,license,python,rust,scoped_cache,parquet,private,private_registry_test,csharp,php,ruby,mysql,quickjs,mcp,run_inline,smtp,http_trigger --all -- --nocapture --test-threads=10 + DENO_PATH=$(which deno) BUN_PATH=$(which bun) NODE_BIN_PATH=$(which node) GO_PATH=$(which go) UV_PATH=$(which uv) PHP_PATH=$(which php) COMPOSER_PATH=$(which composer) RUBY_PATH=$(which ruby) RUBY_BUNDLE_PATH=$(which bundle) RUBY_GEM_PATH=$(which gem) POWERSHELL_PATH=$(which pwsh) DOTNET_PATH=$(which dotnet) cargo test --features enterprise,deno_core,duckdb,license,python,rust,scoped_cache,parquet,private,private_registry_test,csharp,php,ruby,mysql,quickjs,mcp,run_inline,smtp,http_trigger,openidconnect --all -- --nocapture --test-threads=10 diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index d420ff1f00..475d7860d8 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -27,7 +27,7 @@ jobs: DATABASE_URL: postgres://postgres:changeme@postgres:5432/windmill LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }} WORKER_GROUP: main - WORKER_TAGS: deno,bun,go,python3,bash,dependency,flow,nativets + WORKER_TAGS: bun,go,python3,bash,dependency,flow,nativets options: >- --pull always --health-interval 10s --health-timeout 5s --health-retries 5 --health-cmd "curl @@ -111,7 +111,7 @@ jobs: DATABASE_URL: postgres://postgres:changeme@postgres:5432/windmill LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }} WORKER_GROUP: main - WORKER_TAGS: deno,bun,go,python3,bash,dependency,flow,nativets + WORKER_TAGS: bun,go,python3,bash,dependency,flow,nativets options: >- --pull always --health-interval 10s --health-timeout 5s --health-retries 5 --health-cmd "curl @@ -125,7 +125,7 @@ jobs: LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }} MODE: worker WORKER_GROUP: main - WORKER_TAGS: deno,bun,go,python3,bash,dependency,flow,nativets + WORKER_TAGS: bun,go,python3,bash,dependency,flow,nativets options: >- --pull always @@ -136,7 +136,7 @@ jobs: LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }} MODE: worker WORKER_GROUP: main - WORKER_TAGS: deno,bun,go,python3,bash,dependency,flow,nativets + WORKER_TAGS: bun,go,python3,bash,dependency,flow,nativets options: >- --pull always @@ -147,7 +147,7 @@ jobs: LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }} MODE: worker WORKER_GROUP: main - WORKER_TAGS: deno,bun,go,python3,bash,dependency,flow,nativets + WORKER_TAGS: bun,go,python3,bash,dependency,flow,nativets options: >- --pull always @@ -188,7 +188,7 @@ jobs: DATABASE_URL: postgres://postgres:changeme@postgres:5432/windmill LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }} WORKER_GROUP: main - WORKER_TAGS: deno,bun,go,python3,bash,dependency,flow,nativets + WORKER_TAGS: bun,go,python3,bash,dependency,flow,nativets options: >- --pull always --health-interval 10s --health-timeout 5s --health-retries 5 --health-cmd "curl @@ -202,7 +202,7 @@ jobs: LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }} MODE: worker WORKER_GROUP: main - WORKER_TAGS: deno,bun,go,python3,bash,dependency,flow,nativets + WORKER_TAGS: bun,go,python3,bash,dependency,flow,nativets options: >- --pull always @@ -213,7 +213,7 @@ jobs: LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }} MODE: worker WORKER_GROUP: main - WORKER_TAGS: deno,bun,go,python3,bash,dependency,flow,nativets + WORKER_TAGS: bun,go,python3,bash,dependency,flow,nativets options: >- --pull always @@ -224,7 +224,7 @@ jobs: LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }} MODE: worker WORKER_GROUP: main - WORKER_TAGS: deno,bun,go,python3,bash,dependency,flow,nativets + WORKER_TAGS: bun,go,python3,bash,dependency,flow,nativets options: >- --pull always @@ -235,7 +235,7 @@ jobs: LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }} MODE: worker WORKER_GROUP: main - WORKER_TAGS: deno,bun,go,python3,bash,dependency,flow,nativets + WORKER_TAGS: bun,go,python3,bash,dependency,flow,nativets options: >- --pull always @@ -246,7 +246,7 @@ jobs: LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }} MODE: worker WORKER_GROUP: main - WORKER_TAGS: deno,bun,go,python3,bash,dependency,flow,nativets + WORKER_TAGS: bun,go,python3,bash,dependency,flow,nativets options: >- --pull always @@ -257,7 +257,7 @@ jobs: LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }} MODE: worker WORKER_GROUP: main - WORKER_TAGS: deno,bun,go,python3,bash,dependency,flow,nativets + WORKER_TAGS: bun,go,python3,bash,dependency,flow,nativets options: >- --pull always @@ -268,7 +268,7 @@ jobs: LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }} MODE: worker WORKER_GROUP: main - WORKER_TAGS: deno,bun,go,python3,bash,dependency,flow,nativets + WORKER_TAGS: bun,go,python3,bash,dependency,flow,nativets options: >- --pull always steps: @@ -309,7 +309,7 @@ jobs: DATABASE_URL: postgres://postgres:changeme@postgres:5432/windmill LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }} WORKER_GROUP: main - WORKER_TAGS: deno,bun,go,python3,bash,dependency,flow,nativets + WORKER_TAGS: bun,go,python3,bash,dependency,flow,nativets options: >- --pull always --health-interval 10s --health-timeout 5s --health-retries 5 --health-cmd "curl diff --git a/.github/workflows/npm_on_release.yml b/.github/workflows/npm_on_release.yml index 631ab53153..a2bd974daa 100644 --- a/.github/workflows/npm_on_release.yml +++ b/.github/workflows/npm_on_release.yml @@ -18,16 +18,19 @@ jobs: env: NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} publish_chat_sdk: - runs-on: ubicloud-standard-8 + # Published by npm trusted publishing (OIDC), which npm accepts only from + # GitHub-hosted runners and npm >= 11.5.1: keep both, there is no token to fall back on. + runs-on: ubuntu-latest + permissions: + contents: read + id-token: write steps: - uses: actions/checkout@v4 - uses: actions/setup-node@v7 with: - node-version: "20.x" + node-version: "24.x" registry-url: "https://registry.npmjs.org" - run: cd chat-sdk && npm ci && npm run build && npm publish --access public - env: - NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} publish_cli: runs-on: ubicloud-standard-8 steps: diff --git a/.release-please-manifest.json b/.release-please-manifest.json index afbaa8e7b0..fdce702331 100644 --- a/.release-please-manifest.json +++ b/.release-please-manifest.json @@ -1,3 +1,3 @@ { - ".": "1.821.0" + ".": "1.830.0" } diff --git a/AGENTS.md b/AGENTS.md index 8451a0a600..a57215cde5 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -33,12 +33,16 @@ Open-source platform for internal tools, workflows, API integrations, background administers behind `external_instance` data tables and Ducklake catalogs: its invariants (one lifecycle lock, managed-object markers, the setup gate, per-cluster roles, fork copy ownership) and how to run one locally +- **Data table viewer roles**: `docs/datatable-viewer-roles.md`, on what a connection naming no + role holds: the default role as a floor, where the viewer comes from, why an admin's roles are + not the viewer's, the per-role-set logins and the order their rows and logins are written in - **Enterprise**: `docs/enterprise.md` — EE file conventions and PR workflow - **Operator write rights**: `docs/operator-write-rights.md` — which `operator_settings` flags are enforced rather than cosmetic, and why a right that is granted-unless-withdrawn needs `Option` fields and a jsonb merge rather than a serde default -- **Operator builder rights**: `docs/operator-builder-rights.md` — the workspace setting that lets - operators compose flows, what the composition check must cover, and why it costs a full seat +- **Operator builder rights**: `docs/operator-builder-rights.md` — the workspace settings that let + operators compose flows and full-code apps, what the composition check must cover, which + raw-app endpoint stays closed on purpose, and why either right costs a full seat - **Auth surface**: `docs/auth-surface.md` — credential precedence, session/cache invalidation scope, which token labels email their owner at expiry, how OAuth login matches `login_type`, and that every superadmin route refuses `$WM_TOKEN`. Read before designing anything that creates @@ -90,6 +94,12 @@ are for a plain checkout with nothing running. - **Login**: `admin@windmill.dev` / `changeme` - **Instance settings**: navigate to `/#superadmin-settings` - **Migrations**: use `cargo sqlx migrate add -r ` from `backend/` to create new migrations (never generate timestamps manually) +- **Resource types ship through the hub, not migrations.** A new type, a provider's credential + included, is a `hub//.resource_type.json` in `windmill-labs/windmill-integrations` + and reaches instances by hub sync; a product name its identifier misses (`typesafe` → + "TypeSafe") is that file's `display_name`, not a word in `resourceTypeDisplay.ts`'s map. Only a + type Windmill itself defines and reads as a product object (`ai_skill`, `ai_instruction`) is + also seeded by a migration, so the feature works before a sync. For local testing, insert the row. ### Per-worktree ports and database diff --git a/CHANGELOG.md b/CHANGELOG.md index 8aa66a785d..80939b1b47 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,224 @@ # Changelog +## [1.830.0](https://github.com/windmill-labs/windmill/compare/v1.829.0...v1.830.0) (2026-10-09) + + +### Features + +* add a jobs:run:dependencies token scope for lock generation ([#11643](https://github.com/windmill-labs/windmill/issues/11643)) ([1dc4ed5](https://github.com/windmill-labs/windmill/commit/1dc4ed5e7fb86c9658f1f877e7b92cd9dab5e283)) +* let operators compose full-code apps when the workspace grants it ([#11229](https://github.com/windmill-labs/windmill/issues/11229)) ([02e4697](https://github.com/windmill-labs/windmill/commit/02e4697157112581e9e2a9303180077126db714b)) +* operators get AI sessions ([#11443](https://github.com/windmill-labs/windmill/issues/11443)) ([338c4e9](https://github.com/windmill-labs/windmill/commit/338c4e921dc3185d886c8b040f253285adf20efb)) +* replace the hub_sync script with an opt-in daily server sync ([#11624](https://github.com/windmill-labs/windmill/issues/11624)) ([2b55a8a](https://github.com/windmill-labs/windmill/commit/2b55a8afbc8a86c6cebe97b1cab02c313b986f22)) +* retry transient AI provider failures before output reaches readers ([#11623](https://github.com/windmill-labs/windmill/issues/11623)) ([d3ddef0](https://github.com/windmill-labs/windmill/commit/d3ddef0b7ec4705aa578c4eca24b96e9152485bd)) +* stop offering deno for new scripts, ship it only on full images ([#11648](https://github.com/windmill-labs/windmill/issues/11648)) ([cda7883](https://github.com/windmill-labs/windmill/commit/cda78830fde1b96bd3de2674e23c245b50e3c051)) + + +### Bug Fixes + +* **cli:** count superadmins as able to preserve ownership on push ([#11653](https://github.com/windmill-labs/windmill/issues/11653)) ([70db567](https://github.com/windmill-labs/windmill/commit/70db567a78b1e1bc9e2dcc9ff2139951d778117e)) +* harden app secret validation ([#11655](https://github.com/windmill-labs/windmill/issues/11655)) ([8556a7b](https://github.com/windmill-labs/windmill/commit/8556a7bebe093e0bd61c33d67415fd553f0070ad)) +* open the chat + menu without waiting on the skills and mcp fetches ([#11645](https://github.com/windmill-labs/windmill/issues/11645)) ([1fea1aa](https://github.com/windmill-labs/windmill/commit/1fea1aa6609d644e02bac8d16186f9a2dfcb63cd)) +* say Running on a chat run card only once its job exists ([#11627](https://github.com/windmill-labs/windmill/issues/11627)) ([d046187](https://github.com/windmill-labs/windmill/commit/d046187e3edd01535240c6080b483afe154ae189)) + +## [1.829.0](https://github.com/windmill-labs/windmill/compare/v1.828.0...v1.829.0) (2026-10-09) + + +### Features + +* add an api base url instance setting for displayed urls ([#11628](https://github.com/windmill-labs/windmill/issues/11628)) ([6acf185](https://github.com/windmill-labs/windmill/commit/6acf1858540e83b65b5218c50c98af692092d22e)) +* add Pi template to the AI sandbox picker ([#11619](https://github.com/windmill-labs/windmill/issues/11619)) ([a85d0b7](https://github.com/windmill-labs/windmill/commit/a85d0b76cc494aaa4a484a273ec534bdc2958102)) +* agent nits: multi-select, memory-off notice, agent step label ([#11622](https://github.com/windmill-labs/windmill/issues/11622)) ([c52ba2f](https://github.com/windmill-labs/windmill/commit/c52ba2fd30e19986522420ff02fae224a075b4f6)) +* export worker cpu, memory and occupancy metrics ([#11629](https://github.com/windmill-labs/windmill/issues/11629)) ([5d90f96](https://github.com/windmill-labs/windmill/commit/5d90f96863b64e9bc88ed35429f8068a6a788d1e)) +* let a job see that its worker is draining ([#11612](https://github.com/windmill-labs/windmill/issues/11612)) ([588d846](https://github.com/windmill-labs/windmill/commit/588d8463dfa1c8936b8753b5f0fd794ecb115203)) +* let workspace admins delete other users' drafts ([#11644](https://github.com/windmill-labs/windmill/issues/11644)) ([68376fc](https://github.com/windmill-labs/windmill/commit/68376fc011ef6406a820c69217058316514d3f8d)) +* per-script resource rollup and heaviest scripts table ([#11631](https://github.com/windmill-labs/windmill/issues/11631)) ([442357d](https://github.com/windmill-labs/windmill/commit/442357dbe8db0c52995572acd7385dedf4efdc42)) +* pipeline editor: assets-only view, Fix actions and draft triggers ([#11440](https://github.com/windmill-labs/windmill/issues/11440)) ([3dae553](https://github.com/windmill-labs/windmill/commit/3dae553576ba23b2c1f56e4deac01305c4645c01)) + + +### Bug Fixes + +* advertise the actual signing algorithm in OIDC discovery ([#11637](https://github.com/windmill-labs/windmill/issues/11637)) ([98fc65a](https://github.com/windmill-labs/windmill/commit/98fc65a16bbe2bc9515cdd5443304053c4322274)) +* ask a workspace for its session backups before pushing to it ([#11641](https://github.com/windmill-labs/windmill/issues/11641)) ([2ea846f](https://github.com/windmill-labs/windmill/commit/2ea846f563f5a77cbd3eae2eddee9e38a3700d00)) +* keep instance settings users sub-tabs from being squashed ([#11633](https://github.com/windmill-labs/windmill/issues/11633)) ([fb22e5c](https://github.com/windmill-labs/windmill/commit/fb22e5ce368eba9146e0d51b8fdabb52ca9f68f0)) +* keep raw app deploy panel settings shared with the editor ([#11616](https://github.com/windmill-labs/windmill/issues/11616)) ([0a95ea4](https://github.com/windmill-labs/windmill/commit/0a95ea47facaede2545cc4741d4a70c1c847e11a)) +* load schedules once per page load, poll critical alerts every 30s ([#11639](https://github.com/windmill-labs/windmill/issues/11639)) ([1b4e686](https://github.com/windmill-labs/windmill/commit/1b4e686b2f5de3138df0c3b6ed627ea0fe0a2f7d)) +* name the open workspace settings tab in the page header breadcrumb ([#11617](https://github.com/windmill-labs/windmill/issues/11617)) ([e233908](https://github.com/windmill-labs/windmill/commit/e233908c1e99a246dc7c94fa86ea82c5cbe9d6b3)) +* **python:** add missing import-to-package mappings for common modules ([#11638](https://github.com/windmill-labs/windmill/issues/11638)) ([ab8606f](https://github.com/windmill-labs/windmill/commit/ab8606f37921913ef1410da735abb5742c025225)) +* send hub script completions to the workspace handlers ([#11613](https://github.com/windmill-labs/windmill/issues/11613)) ([d685332](https://github.com/windmill-labs/windmill/commit/d685332771dcdfc718d7a2077fe1cdb3b323909b)) +* set WM_END_USER_EMAIL on direct script and flow runs ([#11636](https://github.com/windmill-labs/windmill/issues/11636)) ([2bdeceb](https://github.com/windmill-labs/windmill/commit/2bdeceba7fb6815cf3150ecb856afb8eb3872e44)) +* store a WAC parent's first start and wall duration ([#11625](https://github.com/windmill-labs/windmill/issues/11625)) ([8f28a58](https://github.com/windmill-labs/windmill/commit/8f28a5860595dbe558e6777a06bb97b6ec6a3e67)) +* sync back volume files rewritten to the same size ([#11620](https://github.com/windmill-labs/windmill/issues/11620)) ([2758920](https://github.com/windmill-labs/windmill/commit/2758920f1182ddd12e6473a644a4a966a435522e)) +* uniform decrypt error and clear stored credentials on logout ([#11632](https://github.com/windmill-labs/windmill/issues/11632)) ([987c321](https://github.com/windmill-labs/windmill/commit/987c321ebcd305df79fa945e941abc2b2e1d3e95)) + +## [1.828.0](https://github.com/windmill-labs/windmill/compare/v1.827.0...v1.828.0) (2026-10-08) + + +### Features + +* add gpt-6.1-sol to the chat, agent steps and weekly ai evals ([#11584](https://github.com/windmill-labs/windmill/issues/11584)) ([8f7e0c7](https://github.com/windmill-labs/windmill/commit/8f7e0c72848b56fa97f6f0141d930c7d15044864)) +* add wmill user list and document post-suspend approvers context ([#11600](https://github.com/windmill-labs/windmill/issues/11600)) ([1c2667c](https://github.com/windmill-labs/windmill/commit/1c2667cd5530925893cf69fd622938b6ffa55d1d)) +* ask before minting the frontend sdk token in the raw app editor ([#11604](https://github.com/windmill-labs/windmill/issues/11604)) ([6e948b0](https://github.com/windmill-labs/windmill/commit/6e948b07d13b85e9251c3a200bf149f167649bc4)) +* improve data table roles and grants UX ([#11562](https://github.com/windmill-labs/windmill/issues/11562)) ([d7d18f0](https://github.com/windmill-labs/windmill/commit/d7d18f00af4e2df8fcb1362c9439d2250d0ce4b6)) +* one page header across routes, and one sidebar for operators and developers ([#11310](https://github.com/windmill-labs/windmill/issues/11310)) ([41162b2](https://github.com/windmill-labs/windmill/commit/41162b270c70ddac6a256bde9acf31a629990e24)) + + +### Bug Fixes + +* add retry to runScriptByPathAsync and runScriptByHashAsync ([#11485](https://github.com/windmill-labs/windmill/issues/11485)) ([3f3b1c2](https://github.com/windmill-labs/windmill/commit/3f3b1c2c308ddaba377db45308711e7c660d8775)) +* describe exponential retry as multiplier × seconds^attempt ([#11546](https://github.com/windmill-labs/windmill/issues/11546)) ([a2e51cd](https://github.com/windmill-labs/windmill/commit/a2e51cdbaafb7d71b8e8bbd818e40c472098f9ba)) +* keep capturing job logs after a non-UTF-8 output line ([#11595](https://github.com/windmill-labs/windmill/issues/11595)) ([0e7d974](https://github.com/windmill-labs/windmill/commit/0e7d974ce9b468353ecb6618dace710704690687)) + + +### Performance Improvements + +* load public raw apps without the loading screen ([#11608](https://github.com/windmill-labs/windmill/issues/11608)) ([c09764f](https://github.com/windmill-labs/windmill/commit/c09764fa6516badddbdbb520af32e5d1541cf996)) + +## [1.827.0](https://github.com/windmill-labs/windmill/compare/v1.826.0...v1.827.0) (2026-10-07) + + +### Features + +* audit instance settings writes, stable audit OTel target ([#11593](https://github.com/windmill-labs/windmill/issues/11593)) ([0792332](https://github.com/windmill-labs/windmill/commit/0792332859a88b1359ea1be64250d38dd071a0a9)) +* default new raw apps to react 19 and ask for the data setup ([#11586](https://github.com/windmill-labs/windmill/issues/11586)) ([818ddd5](https://github.com/windmill-labs/windmill/commit/818ddd5d31bd81507e274dd18022c1cdbd9be842)) +* export queue delay as OTel and Prometheus metrics ([#11588](https://github.com/windmill-labs/windmill/issues/11588)) ([7565349](https://github.com/windmill-labs/windmill/commit/75653496e542550039cbe5c413d8fcb9d395cea8)) +* pause a worker group without stopping its workers ([#11596](https://github.com/windmill-labs/windmill/issues/11596)) ([95c4b30](https://github.com/windmill-labs/windmill/commit/95c4b3096cf10c1fe59de7a1a6876dfb5f02c39f)) +* run AI decision steps on OpenAI's Decisions API ([#11590](https://github.com/windmill-labs/windmill/issues/11590)) ([046df9e](https://github.com/windmill-labs/windmill/commit/046df9e44db2f2fae3059ccd7a3d6c6955ee8971)) + + +### Bug Fixes + +* avoid URLSearchParams.keys() so the frontend check passes ([#11599](https://github.com/windmill-labs/windmill/issues/11599)) ([fc6c84c](https://github.com/windmill-labs/windmill/commit/fc6c84c6974bba1850e90a088fad5460c8c93ba3)) +* let recover: true from the error handler turn nested failures green ([#11559](https://github.com/windmill-labs/windmill/issues/11559)) ([03e484e](https://github.com/windmill-labs/windmill/commit/03e484e59757c0d4d6f74511d9aa5a1e2f94af67)) +* run dependency resolution and build steps under nsjail ([#11609](https://github.com/windmill-labs/windmill/issues/11609)) ([a238d01](https://github.com/windmill-labs/windmill/commit/a238d01ef6a8c285aef6886c974904f24339f7a2)) +* scope the python wheel cache to workspaces with their own registry ([#11597](https://github.com/windmill-labs/windmill/issues/11597)) ([14381ce](https://github.com/windmill-labs/windmill/commit/14381cefee6fb2b0ece52fbf0001c2ce61bde1a1)) +* type snowflake columns in database manager insert and alter forms ([#11578](https://github.com/windmill-labs/windmill/issues/11578)) ([f2fb156](https://github.com/windmill-labs/windmill/commit/f2fb15603b2e0d11bd5d04dd675212a5e4a7b9d7)) + +## [1.826.0](https://github.com/windmill-labs/windmill/compare/v1.825.0...v1.826.0) (2026-10-07) + + +### Features + +* count OIDC signatures and reuse server-side tokens ([#11556](https://github.com/windmill-labs/windmill/issues/11556)) ([e277735](https://github.com/windmill-labs/windmill/commit/e277735788c260239a063063459776ec01f7d907)) +* isolate the raw-app editor preview when the app opts into sandboxing ([#11572](https://github.com/windmill-labs/windmill/issues/11572)) ([fa25397](https://github.com/windmill-labs/windmill/commit/fa253976e213c1053d1832c1d1fb7bedf7ce8c45)) +* job_token_scopes OIDC claim ([#11551](https://github.com/windmill-labs/windmill/issues/11551)) ([d493eff](https://github.com/windmill-labs/windmill/commit/d493eff237f96a353fb6f5715a15bcad61d24433)) +* set up a new app in one modal over its AI session, fix its deploy ([#11582](https://github.com/windmill-labs/windmill/issues/11582)) ([7190185](https://github.com/windmill-labs/windmill/commit/71901852fb3cc40515d6675a0a88cd158e942d56)) + + +### Bug Fixes + +* export getWorkspace and other client helpers as named exports ([#11581](https://github.com/windmill-labs/windmill/issues/11581)) ([bf201be](https://github.com/windmill-labs/windmill/commit/bf201be11e95f0fc3e12353531f25bb3ceb0f8f3)) +* give the dbt workspace settings page a header like its siblings ([#11576](https://github.com/windmill-labs/windmill/issues/11576)) ([38aa4ea](https://github.com/windmill-labs/windmill/commit/38aa4eaa1200db52e67a18ef9a7c5dc4d60a4948)) +* ignore schedule missed-run counters in workspace compare ([#11570](https://github.com/windmill-labs/windmill/issues/11570)) ([eab3dd5](https://github.com/windmill-labs/windmill/commit/eab3dd5a52bc8faf57bb84c85bd74eb20ebd8ec7)) +* keep chat tool cards collapsed the way the user left them ([#11575](https://github.com/windmill-labs/windmill/issues/11575)) ([c71ecad](https://github.com/windmill-labs/windmill/commit/c71ecad6f9cf3727a305b5f9038dcbe6a54398b8)) +* keep run status buttons from selecting the flow input node ([#11579](https://github.com/windmill-labs/windmill/issues/11579)) ([8661797](https://github.com/windmill-labs/windmill/commit/866179730a9798346d3950f27dd625c3358e690a)) +* make every trigger and schedule editor control respect read-only mode ([#11569](https://github.com/windmill-labs/windmill/issues/11569)) ([52a99c3](https://github.com/windmill-labs/windmill/commit/52a99c3047e1add477e36919b15a745f7d2604a0)) +* strip NUL from ai agent output and job args before jsonb writes ([#11568](https://github.com/windmill-labs/windmill/issues/11568)) ([cf4bec6](https://github.com/windmill-labs/windmill/commit/cf4bec611f6df8a45b238afe5b8da3e3a5748bf9)) + +## [1.825.0](https://github.com/windmill-labs/windmill/compare/v1.824.1...v1.825.0) (2026-10-06) + + +### Features + +* choose build with AI or code when creating a raw app ([#11560](https://github.com/windmill-labs/windmill/issues/11560)) ([431f861](https://github.com/windmill-labs/windmill/commit/431f86138815fce432e282e8ac495953c5ecf98a)) +* confine jobs:read to the runnable paths it names ([#11550](https://github.com/windmill-labs/windmill/issues/11550)) ([b371af3](https://github.com/windmill-labs/windmill/commit/b371af31c96e6890fe9223233d2c6e53ddea67d4)) +* open a run as a pushed page on the narrow runs page ([#11541](https://github.com/windmill-labs/windmill/issues/11541)) ([fd8de45](https://github.com/windmill-labs/windmill/commit/fd8de45cc7cfcba2c374dd86f87f4d3e8aec6ded)) +* say a canceled team plan is canceled, not past due ([#11539](https://github.com/windmill-labs/windmill/issues/11539)) ([18e44d3](https://github.com/windmill-labs/windmill/commit/18e44d31742b8d16852b8c6ae663979666e064d7)) + + +### Bug Fixes + +* group raw app edits that carry a file diff in the ai chat ([#11558](https://github.com/windmill-labs/windmill/issues/11558)) ([5912d30](https://github.com/windmill-labs/windmill/commit/5912d300a550a33674fd73740d613d1b434fbba1)) +* honor custom timeout in setup phases and name the limit hit ([#11557](https://github.com/windmill-labs/windmill/issues/11557)) ([4fd4f76](https://github.com/windmill-labs/windmill/commit/4fd4f76d0df740cb3eebed1a0625b970484807fc)) +* search slow runs filters by time windows instead of timing out ([#11549](https://github.com/windmill-labs/windmill/issues/11549)) ([2bf9cfc](https://github.com/windmill-labs/windmill/commit/2bf9cfcd59e1101695160524c31a4da1a41699dc)) +* serve the raw-app preview shell sandboxed, load a blob copy in the editor ([#11554](https://github.com/windmill-labs/windmill/issues/11554)) ([5f92daa](https://github.com/windmill-labs/windmill/commit/5f92daab6d0ecac30dd384e1b0f302af13fdd9ab)) +* simplify the run page AI card to a prompt and a chat button ([#11561](https://github.com/windmill-labs/windmill/issues/11561)) ([8c74f75](https://github.com/windmill-labs/windmill/commit/8c74f75de3fa62b07186ca4f56dae4a7e4758034)) + +## [1.824.1](https://github.com/windmill-labs/windmill/compare/v1.824.0...v1.824.1) (2026-10-05) + + +### Bug Fixes + +* keep monaco's diff engine out of the app shell bundle ([#11542](https://github.com/windmill-labs/windmill/issues/11542)) ([0908dee](https://github.com/windmill-labs/windmill/commit/0908dee897e0f87b97418a23ede3662d96270340)) + +## [1.824.0](https://github.com/windmill-labs/windmill/compare/v1.823.1...v1.824.0) (2026-10-05) + + +### Features + +* add an AI decision flow step (TypeSafe's Jev, Cloudflare's Clef) ([#11474](https://github.com/windmill-labs/windmill/issues/11474)) ([3cba8a7](https://github.com/windmill-labs/windmill/commit/3cba8a7e4cca3ef20ba73a7dad79d892bdc9fed8)) +* add notion oauth provider ([#11528](https://github.com/windmill-labs/windmill/issues/11528)) ([8b2dc1e](https://github.com/windmill-labs/windmill/commit/8b2dc1e46c7f4e3859263f18b686ce6f3ce6fd30)) +* **ai-chat:** show per-file diffs for raw app edits ([#11521](https://github.com/windmill-labs/windmill/issues/11521)) ([ec0883a](https://github.com/windmill-labs/windmill/commit/ec0883a627b06b6ba2a393d5cf66ddf6ebc4cd0d)) +* bill team-plan service accounts as half a seat ([#11298](https://github.com/windmill-labs/windmill/issues/11298)) ([3aee181](https://github.com/windmill-labs/windmill/commit/3aee1815746f0116f5662f162ad1e234717af90d)) +* card layout and load-more for the audit logs page ([#11507](https://github.com/windmill-labs/windmill/issues/11507)) ([af1a29a](https://github.com/windmill-labs/windmill/commit/af1a29a3cb33386f61a759ec0dd5eec0ccefefc2)) +* offer audit-log and drive-metadata scopes for google oauth connects ([#11527](https://github.com/windmill-labs/windmill/issues/11527)) ([c4e7c71](https://github.com/windmill-labs/windmill/commit/c4e7c714e455f958d9a927c4bc4eb23256324780)) +* offer slack read scopes for files, user groups, emoji and dms ([#11526](https://github.com/windmill-labs/windmill/issues/11526)) ([b5fda7d](https://github.com/windmill-labs/windmill/commit/b5fda7d5af7105fde69beb68410984350af2ac7e)) + + +### Bug Fixes + +* bound s3 ai attachment downloads before encoding ([#11534](https://github.com/windmill-labs/windmill/issues/11534)) ([92fbfc0](https://github.com/windmill-labs/windmill/commit/92fbfc03d219544c16e8f7db47a547f9ff402c27)) +* collapse queued and running chat tool calls into a waiting row ([#11523](https://github.com/windmill-labs/windmill/issues/11523)) ([4f30982](https://github.com/windmill-labs/windmill/commit/4f30982ce3f7abc64a31890f8e88c31c1c2e0803)) +* disable postgres triggers whose replication slot was invalidated ([#11536](https://github.com/windmill-labs/windmill/issues/11536)) ([85248ee](https://github.com/windmill-labs/windmill/commit/85248eee786b633def40c42b6007fc7b907fb09e)) +* evaluate field conditions and step test inputs in QuickJS ([#11487](https://github.com/windmill-labs/windmill/issues/11487)) ([c962d37](https://github.com/windmill-labs/windmill/commit/c962d3766b1a250ba960e413556890d5919ebc94)) +* flush proxied responses before the dev supervisor closes the client ([#11530](https://github.com/windmill-labs/windmill/issues/11530)) ([cefb195](https://github.com/windmill-labs/windmill/commit/cefb1953764e714d79556e2f8065cfbec78acb86)) +* label the default slack scopes and list them first in the connect dialog ([#11529](https://github.com/windmill-labs/windmill/issues/11529)) ([e76781c](https://github.com/windmill-labs/windmill/commit/e76781cc2709fda08403e14c1a0a01957e430db1)) +* run each ai eval model in its own job and let the docs tools reach the backend ([#11533](https://github.com/windmill-labs/windmill/issues/11533)) ([222dad1](https://github.com/windmill-labs/windmill/commit/222dad1d8cc9cbb6925fdff709e00a49bb63bc4a)) +* show the flow row edit button to builder operators ([#11535](https://github.com/windmill-labs/windmill/issues/11535)) ([6c4e34c](https://github.com/windmill-labs/windmill/commit/6c4e34cd0dd9b4329084214c40695f5915947cc2)) + +## [1.823.1](https://github.com/windmill-labs/windmill/compare/v1.823.0...v1.823.1) (2026-10-05) + + +### Bug Fixes + +* python results with a KeyError-raising __getattr__ fail the job ([#11517](https://github.com/windmill-labs/windmill/issues/11517)) ([3fff1bd](https://github.com/windmill-labs/windmill/commit/3fff1bd3dcedce22c58d988ad2e725e3d4e96e0e)) +* stream the ai chat compaction request ([#11514](https://github.com/windmill-labs/windmill/issues/11514)) ([289d3c3](https://github.com/windmill-labs/windmill/commit/289d3c3941c2d9997bf0476bf9bfdfd948542c55)) + +## [1.823.0](https://github.com/windmill-labs/windmill/compare/v1.822.0...v1.823.0) (2026-10-04) + + +### Features + +* redesign the service logs page ([#11506](https://github.com/windmill-labs/windmill/issues/11506)) ([0f65df2](https://github.com/windmill-labs/windmill/commit/0f65df24d2cabd9d8daa71c103969e6392e37b4f)) + + +### Bug Fixes + +* apply the pg keepalive settings to the postgres trigger connection ([#11512](https://github.com/windmill-labs/windmill/issues/11512)) ([33f6de0](https://github.com/windmill-labs/windmill/commit/33f6de0cd20f45ff56770f946a4731e5b13bf677)) +* bound the datatable migration lock wait and reap a dead holder ([#11510](https://github.com/windmill-labs/windmill/issues/11510)) ([711e08c](https://github.com/windmill-labs/windmill/commit/711e08c1d20a326142235c83d8805b7990bf6e6d)) +* bound the pg cached-connection reset probe so a vanished server fails fast ([#11505](https://github.com/windmill-labs/windmill/issues/11505)) ([d4a423f](https://github.com/windmill-labs/windmill/commit/d4a423f567e1fb5e21016a2be61d858ab61a4d4f)) +* cancel an abandoned pg query and close failed connections cleanly ([#11511](https://github.com/windmill-labs/windmill/issues/11511)) ([26856b8](https://github.com/windmill-labs/windmill/commit/26856b8061c28eade191c7d85a32b74ca44c4fea)) +* delete a flow chat turn with its flow run, not its step jobs ([#11477](https://github.com/windmill-labs/windmill/issues/11477)) ([35ec16d](https://github.com/windmill-labs/windmill/commit/35ec16d12728531ab90632116913e98f80d1cc05)) +* drop a fork's datatable database despite connected sessions ([#11509](https://github.com/windmill-labs/windmill/issues/11509)) ([fa419af](https://github.com/windmill-labs/windmill/commit/fa419af9b3f7d65a607bb8ea2bcea58223f743c4)) + +## [1.822.0](https://github.com/windmill-labs/windmill/compare/v1.821.0...v1.822.0) (2026-10-03) + + +### Features + +* add a cancel-only jobs:cancel token scope, optionally path-scoped ([#11479](https://github.com/windmill-labs/windmill/issues/11479)) ([4bc7e0d](https://github.com/windmill-labs/windmill/commit/4bc7e0d7bad22e9e1a3532ca747ed863f90213ba)) +* add an instance setting routing all dependency jobs to one tag ([#11486](https://github.com/windmill-labs/windmill/issues/11486)) ([3d1d249](https://github.com/windmill-labs/windmill/commit/3d1d2495567274b4c48babb374e7716c4ed24ecc)) +* add trigger, tag, run-as and digest claims to job OIDC tokens ([#11481](https://github.com/windmill-labs/windmill/issues/11481)) ([4beb1f9](https://github.com/windmill-labs/windmill/commit/4beb1f9420524425c354e82729e4e35c5bf65451)) +* log pg job progress, warn on stalls, detect dead db connections ([#11491](https://github.com/windmill-labs/windmill/issues/11491)) ([d722772](https://github.com/windmill-labs/windmill/commit/d7227726be2b082bbfb204abb75a3ff081db77de)) +* read the OIDC signing key from a file and rotate it ([#11482](https://github.com/windmill-labs/windmill/issues/11482)) ([d949484](https://github.com/windmill-labs/windmill/commit/d949484e7ba9911cf128387e096f78c917fb3f35)) +* replace the AI input filling toggle with an additional prompt ([#11501](https://github.com/windmill-labs/windmill/issues/11501)) ([e952298](https://github.com/windmill-labs/windmill/commit/e952298f84d04cbb11f0c44c32435f0b711bf3bf)) +* restricted job tokens per script and flow ([#11484](https://github.com/windmill-labs/windmill/issues/11484)) ([e7fc1b2](https://github.com/windmill-labs/windmill/commit/e7fc1b2e2e9b358b414d0ca0eaf2753e142dded7)) + + +### Bug Fixes + +* keep a resource default set in the schema when the script is redeployed ([#11495](https://github.com/windmill-labs/windmill/issues/11495)) ([68f8f19](https://github.com/windmill-labs/windmill/commit/68f8f19b33768451ebab267b36f931a6f386c678)) +* let instances withhold signing secrets from the settings API ([#11483](https://github.com/windmill-labs/windmill/issues/11483)) ([99e96d3](https://github.com/windmill-labs/windmill/commit/99e96d3f781decda8fed2cbd5409cbc5c3520eaa)) +* let legacy draft-only items be discarded from the home page ([#11488](https://github.com/windmill-labs/windmill/issues/11488)) ([98274a7](https://github.com/windmill-labs/windmill/commit/98274a73361618ff37a929584e5b5f519b71a912)) +* never reuse a pg connection a script left inside a transaction ([#11497](https://github.com/windmill-labs/windmill/issues/11497)) ([b7f74cd](https://github.com/windmill-labs/windmill/commit/b7f74cd56294cdb7052e311c5a68de9ee0bb4122)) +* read inet and cidr results with their prefix, including arrays ([#11476](https://github.com/windmill-labs/windmill/issues/11476)) ([ae093b7](https://github.com/windmill-labs/windmill/commit/ae093b7666cefc3777be0e45144bb272732846d0)) +* stop the pg executor cache from pinning a pooler slot ([#11490](https://github.com/windmill-labs/windmill/issues/11490)) ([eed7e7d](https://github.com/windmill-labs/windmill/commit/eed7e7d9e6bb375daebbb323b1288c36a278f977)) +* unstick postgres jobs on large results with custom-typed columns ([#11475](https://github.com/windmill-labs/windmill/issues/11475)) ([14801fd](https://github.com/windmill-labs/windmill/commit/14801fdd68407add824bfa2c679c0e8cb4788610)) + + +### Performance Improvements + +* keep a pg executor connection per database ([#11492](https://github.com/windmill-labs/windmill/issues/11492)) ([013b302](https://github.com/windmill-labs/windmill/commit/013b302d484b967443cbf747fb9f2ecbb1145033)) + ## [1.821.0](https://github.com/windmill-labs/windmill/compare/v1.820.0...v1.821.0) (2026-10-01) diff --git a/Dockerfile b/Dockerfile index 8d229dba56..c6a4ccc55f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -83,7 +83,6 @@ COPY /python-client/docs/ /frontend/static/pydocs/ RUN npm run generate-backend-client ENV NODE_OPTIONS "--max-old-space-size=8192" -ARG VITE_BASE_URL "" # Must be declared for the build-arg to reach the bundle. See frontend/svelte.config.js. ARG WM_BUILD_VERSION="" # Read more about macro in docker/dev.nu @@ -142,10 +141,10 @@ FROM ${DEBIAN_IMAGE} ARG TARGETPLATFORM ARG POWERSHELL_VERSION=7.5.0 -ARG KUBECTL_VERSION=1.36.2 +ARG KUBECTL_VERSION=1.37.1 ARG HELM_VERSION=3.21.4 # NOTE: If changing, also change go version in workspace dependencies template at WorkspaceDependenciesEditor.svelte -ARG GO_VERSION=1.26.8 +ARG GO_VERSION=1.27.1 ARG APP=/usr/src/app ARG WITH_POWERSHELL=true ARG WITH_KUBECTL=true @@ -242,7 +241,7 @@ ENV PATH="${PATH}:/usr/local/go/bin" ENV GO_PATH=/usr/local/go/bin/go # Install UV -RUN curl --proto '=https' --tlsv1.2 -LsSf https://github.com/astral-sh/uv/releases/download/0.11.24/uv-installer.sh | sh && mv /root/.local/bin/uv /usr/local/bin/uv +RUN curl --proto '=https' --tlsv1.2 -LsSf https://github.com/astral-sh/uv/releases/download/0.13.0/uv-installer.sh | sh && mv /root/.local/bin/uv /usr/local/bin/uv # Preinstall python runtimes to temp build location (will copy with world-writable perms later) # --compile-bytecode precompiles the stdlib to .pyc so jobs don't recompile it on every run @@ -287,8 +286,6 @@ COPY --from=builder /frontend/build /static_frontend COPY --from=builder /windmill/target/release/windmill ${APP}/windmill COPY --from=windmill_duckdb_ffi_internal_builder /windmill-duckdb-ffi-internal/target/release/libwindmill_duckdb_ffi_internal.so ${APP}/libwindmill_duckdb_ffi_internal.so -COPY --from=denoland/deno:2.2.1 --chmod=755 /usr/bin/deno /usr/bin/deno - COPY --from=oven/bun:1.4.0 /usr/local/bin/bun /usr/bin/bun # Install windmill CLI diff --git a/README.md b/README.md index 87fc66f399..e9ed6381f1 100644 --- a/README.md +++ b/README.md @@ -150,7 +150,7 @@ You can run scripts locally by passing the right environment variables for the ` - **Frontend**: Svelte 5 - **Sandboxing**: [nsjail](https://github.com/google/nsjail) and PID namespace isolation - **Runtimes**: - - TypeScript/JavaScript: Bun (default) and Deno + - TypeScript/JavaScript: Bun (default); Deno only on the `windmill-full` images - Python: python3 with uv for dependency management - Go, Bash, PowerShell, PHP, Rust, C#, Java, Ansible @@ -318,7 +318,7 @@ running options. This will also avoid compile time issue with sqlx's `query!` macro. 2. (optional, linux only) Install [nsjail](https://github.com/google/nsjail) and have it accessible in your PATH -3. Install bun, deno and python3 (+ any languages you want to use), have the bins at `/usr/bin/bun`,`/usr/bin/deno`, and +3. Install bun and python3 (+ any languages you want to use), have the bins at `/usr/bin/bun` and `/usr/local/bin/python3` or set the corresponding environment variables. 4. (optional) Install the [lld linker](https://lld.llvm.org/) 5. Go to `frontend/`: diff --git a/ai_evals/README.md b/ai_evals/README.md index bff4294e52..44870ba2a2 100644 --- a/ai_evals/README.md +++ b/ai_evals/README.md @@ -89,7 +89,21 @@ Notes: - the command also prints accepted alias spellings such as `gpt-4o`, `gpt-55`, `claude-opus-4.6`, and `claude-haiku-4.5` - frontend modes (`flow`, `script`, `app`, `global`) can use Anthropic, OpenAI, Gemini, and DeepSeek-backed aliases -- `cli` mode always uses the Anthropic agent SDK, so only Anthropic aliases are valid there +- `cli` mode runs a coding agent over the workspace, chosen by the alias's `cli` entry; aliases without one are rejected there: + + | Agent | Models | Label | Needs | + | --- | --- | --- | --- | + | Claude Code (`@anthropic-ai/claude-agent-sdk`) | Claude aliases | `anthropic:` | `ANTHROPIC_API_KEY` | + | Claude Code on DeepSeek's Anthropic-compatible endpoint | `deepseek-v4-pro` | `claude-code:` | `DEEPSEEK_API_KEY` | + | Codex (`codex exec --json`) | `gpt-6.1-sol`, `gpt-6-astra` | `codex:` | `codex` on `PATH` (CI pins 0.159.3), `OPENAI_API_KEY` | + + Codex runs with a throwaway `HOME`, so your own `~/.codex` config and skills stay out of the run. It has no + Skill tool: it finds `.agents/skills` on its own and loads a skill by reading its `SKILL.md`, so the trace + records any command that touches `.agents/skills//SKILL.md` as a `Skill` call to ``, just + before that command, or just after it when the command mutates something before reading the skill. Its shell commands are recorded as `Bash` and its patches as `Edit`, so the + `invokes skill … before first mutation` checks compare the same things for both agents. Codex has no + turn cap; a run stops at `maxTurns` rounds of tool calls (parallel calls count once) and fails, as a + Claude Code run does at its cap. - the judge model is separate and currently defaults to `claude-sonnet-5-5`; use `--skip-judge` for deterministic-only runs ## Case Format @@ -164,6 +178,12 @@ the decrypted value, exactly as against a real backend. The chat's read path pas Seed a recognizable secret (the existing fixture uses `sk_live_do_not_leak_me`) and assert it via `valueExcludes` to catch a leak. +`runtime.userAnswers` scripts the user's side of `askUserQuestion` in a global case: one +answer per question, in the order they are asked, and a question past the last answer is +dismissed, so `userAnswers: []` dismisses every question. Without it the tool reports that +it cannot ask, and the model usually carries on with its own guess, which makes "asks +before creating anything" impossible to assert. + `toolExpect.toolCallArgs` entries support `sharedByAtLeast: `: at least `n` recorded calls to that tool must carry the same non-blank string in the field. Use it for calls that have to share an identifier, like two test runs of one chat conversation. diff --git a/ai_evals/adapters/cli/codex.test.ts b/ai_evals/adapters/cli/codex.test.ts new file mode 100644 index 0000000000..fdeda1d26b --- /dev/null +++ b/ai_evals/adapters/cli/codex.test.ts @@ -0,0 +1,71 @@ +import { describe, expect, it } from "bun:test"; +import { chmod, mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { runCodex, toToolInvocations, unwrapShellCommand } from "./codex"; + +describe("unwrapShellCommand", () => { + it("strips the login-shell wrapper Codex reports", () => { + expect(unwrapShellCommand(`/usr/bin/zsh -lc 'cat .agents/skills/x/SKILL.md'`)).toBe( + "cat .agents/skills/x/SKILL.md" + ); + expect(unwrapShellCommand(`/bin/bash -lc "printf 'hi' > \\"a b\\""`)).toBe( + `printf 'hi' > "a b"` + ); + expect(unwrapShellCommand(`/bin/bash -lc 'echo '\\''x'\\'''`)).toBe(`echo 'x'`); + expect(unwrapShellCommand(`bash -lc 'wmill sync push'`)).toBe("wmill sync push"); + expect(unwrapShellCommand(`/bin/bash -lc 'rg -g '"'"'!x'"'"' f'`)).toBe(`rg -g '!x' f`); + }); +}); + +describe("toToolInvocations", () => { + it("records reading a SKILL.md as a Skill call before the command", () => { + const tools = toToolInvocations({ + type: "command_execution", + command: `/bin/bash -lc 'sed -n 1,200p .agents/skills/write-script-bun/SKILL.md && mkdir f'`, + }); + expect(tools.map((tool) => [tool.tool, tool.input])).toEqual([ + ["Skill", { skill: "write-script-bun" }], + ["Bash", { command: "sed -n 1,200p .agents/skills/write-script-bun/SKILL.md && mkdir f" }], + ]); + }); + + it("records a skill read after a mutation earlier in the same command", () => { + const tools = toToolInvocations({ + type: "command_execution", + command: `/bin/bash -lc 'printf main > f/a.ts && cat .agents/skills/write-script-bun/SKILL.md'`, + }); + expect(tools.map((tool) => tool.tool)).toEqual(["Bash", "Skill"]); + }); + + it("maps a patch to Edit so it counts as a mutation", () => { + expect( + toToolInvocations({ type: "file_change", changes: [{ path: "a.ts", kind: "add" }] })[0] + ).toMatchObject({ tool: "Edit", input: { changes: [{ path: "a.ts", kind: "add" }] } }); + }); +}); + +describe("runCodex", () => { + it("keeps parallel tool calls in the order they started", async () => { + const dir = await mkdtemp(join(tmpdir(), "fake-codex-")); + const events = [ + { type: "item.started", item: { id: "a", type: "command_execution", command: "mkdir f" } }, + { type: "item.started", item: { id: "b", type: "command_execution", command: "cat .agents/skills/s/SKILL.md" } }, + { type: "item.completed", item: { id: "b", type: "command_execution", command: "cat .agents/skills/s/SKILL.md" } }, + { type: "item.completed", item: { id: "a", type: "command_execution", command: "mkdir f" } }, + { type: "item.completed", item: { id: "c", type: "agent_message", text: "done" } }, + { type: "turn.completed", usage: { input_tokens: 10, output_tokens: 2 } }, + ]; + const lines = events.map((event) => `printf '%s\\n' '${JSON.stringify(event)}'`).join("\n"); + await writeFile(join(dir, "codex"), `#!/bin/sh\ncat > /dev/null\n${lines}\n`); + await chmod(join(dir, "codex"), 0o755); + try { + const result = await runCodex("p", dir, 12, "m", { PATH: `${dir}:${process.env.PATH}` }); + expect(result.toolsUsed.map((tool) => tool.tool)).toEqual(["Bash", "Skill", "Bash"]); + expect(result.output).toBe("done"); + expect(result.tokenUsage).toEqual({ prompt: 10, completion: 2, total: 12 }); + } finally { + await rm(dir, { recursive: true, force: true }); + } + }); +}); diff --git a/ai_evals/adapters/cli/codex.ts b/ai_evals/adapters/cli/codex.ts new file mode 100644 index 0000000000..b49c441556 --- /dev/null +++ b/ai_evals/adapters/cli/codex.ts @@ -0,0 +1,241 @@ +import { spawn } from "node:child_process"; +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { createInterface } from "node:readline"; +import type { BenchmarkTokenUsage, CliToolInvocation } from "../../core/types"; +import { isLikelyMutatingBashCommand } from "./mutation"; + +export interface AgentRunResult { + output: string; + toolsUsed: CliToolInvocation[]; + assistantMessageCount: number; + tokenUsage: BenchmarkTokenUsage | null; + finalContextTokens: number | null; +} + +interface CodexItem { + type?: string; + text?: string; + command?: string; + changes?: { path?: string; kind?: string }[]; + [key: string]: unknown; +} + +interface CodexEvent { + type?: string; + item?: CodexItem; + usage?: { input_tokens?: number; output_tokens?: number }; + message?: string; + error?: { message?: string }; +} + +const SKILL_FILE_PATTERN = /\.agents\/skills\/([^/\s'"`*?[]+)\/SKILL\.md/g; + +// Drives `codex exec` and maps its events onto the Claude Code tool vocabulary +// the validators read: shell commands become `Bash`, patches `Edit`, and +// reading `.agents/skills//SKILL.md` (how Codex loads a skill) becomes a +// `Skill` call recorded just before that command. +export async function runCodex( + prompt: string, + cwd: string, + maxTurns: number, + model: string, + env: Record +): Promise { + // A fresh HOME keeps the user's ~/.codex config and ~/.agents/skills out of the run. + const home = await mkdtemp(join(tmpdir(), "wmill-cli-benchmark-codex-home-")); + const result: AgentRunResult = { + output: "", + toolsUsed: [], + assistantMessageCount: 0, + tokenUsage: null, + finalContextTokens: null, + }; + const messages: string[] = []; + let failure: string | null = null; + let lastError: string | null = null; + let stderr = ""; + + try { + const child = spawn( + "codex", + [ + "exec", + "--json", + "--ephemeral", + "--skip-git-repo-check", + "--dangerously-bypass-approvals-and-sandbox", + "--model", + model, + "--cd", + cwd, + "-", + ], + { + cwd, + env: { + ...env, + HOME: home, + CODEX_HOME: home, + CODEX_API_KEY: env.OPENAI_API_KEY ?? "", + }, + stdio: ["pipe", "pipe", "pipe"], + } + ); + // Without this, a missing `codex` binary leaves the stdout loop waiting forever. + await new Promise((resolve, reject) => { + child.once("spawn", resolve); + child.once("error", reject); + }); + child.stdin.on("error", () => {}); + child.stdin.end(prompt); + child.stderr.on("data", (chunk) => { + stderr = (stderr + chunk).slice(-4000); + }); + const exited = new Promise((resolve, reject) => { + child.on("error", reject); + child.on("close", resolve); + }); + + // Codex has no turn cap. A turn starts when the agent issues tool calls with + // none in flight, so parallel calls count once, as they do for Claude Code. + // Claude Code's cap includes the final reply, so the maxTurns-th tool round is + // already over it. + // ponytail: parallel calls that finish before the next one starts count as + // separate turns; count model calls if Codex ever reports them. + let turns = 0; + let inFlight = 0; + let capped = false; + // Parallel calls complete out of order; the trace keeps the order they started in. + const slots = new Map(); + const ordered: CliToolInvocation[][] = []; + const slotFor = (item: CodexItem) => { + const id = typeof item.id === "string" ? item.id : undefined; + if (id !== undefined && slots.has(id)) { + return slots.get(id)!; + } + ordered.push([]); + if (id !== undefined) { + slots.set(id, ordered.length - 1); + } + return ordered.length - 1; + }; + + for await (const line of createInterface({ input: child.stdout })) { + const event = parseEvent(line); + if (!event) { + continue; + } + const item = event.item; + + if (event.type === "item.started" && item && isToolItem(item)) { + if (inFlight === 0) { + turns += 1; + } + inFlight += 1; + slotFor(item); + if (turns >= maxTurns && !capped) { + capped = true; + child.kill("SIGTERM"); + } + } else if (event.type === "item.completed" && item) { + if (item.type === "agent_message") { + result.assistantMessageCount += 1; + if (typeof item.text === "string") { + messages.push(item.text); + } + } else if (isToolItem(item)) { + inFlight = Math.max(0, inFlight - 1); + ordered[slotFor(item)] = toToolInvocations(item); + } + } else if (event.type === "turn.completed" && event.usage) { + const prompt = event.usage.input_tokens ?? 0; + const completion = event.usage.output_tokens ?? 0; + result.tokenUsage = { prompt, completion, total: prompt + completion }; + } else if (event.type === "turn.failed") { + failure = event.error?.message ?? JSON.stringify(event); + } else if (event.type === "error") { + // Also sent for retries Codex recovers from ("Reconnecting... 1/5"). + lastError = event.message ?? JSON.stringify(event); + } + } + + result.toolsUsed = ordered.flat(); + const code = await exited; + if (failure) { + throw new Error(`codex: ${failure}`); + } + // Fail the attempt, as Claude Code does at its cap. + if (capped) { + throw new Error(`codex: reached maximum number of turns (${maxTurns})`); + } + if (code !== 0) { + throw new Error(`codex exited with ${code}: ${lastError ?? stderr.trim()}`); + } + } finally { + await rm(home, { recursive: true, force: true }); + } + + result.output = messages.join("\n"); + return result; +} + +export function toToolInvocations(item: CodexItem): CliToolInvocation[] { + const timestamp = Date.now(); + + if (item.type === "command_execution" && typeof item.command === "string") { + const command = unwrapShellCommand(item.command); + // A skill read counts as preceding the command only when nothing before it in + // the command mutates; otherwise `printf x > f && cat SKILL.md` would pass a + // skill-before-first-mutation check. + const before: string[] = []; + const after: string[] = []; + for (const match of command.matchAll(SKILL_FILE_PATTERN)) { + const skill = match[1]!; + if (before.includes(skill) || after.includes(skill)) { + continue; + } + (isLikelyMutatingBashCommand(command.slice(0, match.index)) ? after : before).push(skill); + } + const skillCall = (skill: string) => ({ tool: "Skill", input: { skill }, timestamp }); + return [ + ...before.map(skillCall), + { tool: "Bash", input: { command }, timestamp }, + ...after.map(skillCall), + ]; + } + + if (item.type === "file_change") { + return [{ tool: "Edit", input: { changes: item.changes ?? [] }, timestamp }]; + } + + const { id: _id, type, status: _status, ...input } = item; + return [{ tool: String(type), input, timestamp }]; +} + +// Codex reports commands as run through the user's login shell, e.g. +// `/usr/bin/zsh -lc 'cat SKILL.md'`. +export function unwrapShellCommand(command: string): string { + const match = command.match(/^(?:\S*\/)?(?:ba|z)?sh -l?c (['"])([\s\S]*)\1$/); + if (!match) { + return command; + } + const body = match[2]!; + return match[1] === "'" + ? body.replaceAll(`'\\''`, "'").replaceAll(`'"'"'`, "'") + : body.replace(/\\(["\\$`])/g, "$1"); +} + +function isToolItem(item: CodexItem): boolean { + return !["agent_message", "reasoning", "todo_list", "error"].includes(item.type ?? ""); +} + +function parseEvent(line: string): CodexEvent | null { + try { + const value = JSON.parse(line); + return value && typeof value === "object" ? (value as CodexEvent) : null; + } catch { + return null; + } +} diff --git a/ai_evals/adapters/cli/mutation.ts b/ai_evals/adapters/cli/mutation.ts new file mode 100644 index 0000000000..05b29d68d7 --- /dev/null +++ b/ai_evals/adapters/cli/mutation.ts @@ -0,0 +1,10 @@ +export function isLikelyMutatingBashCommand(command: string): boolean { + return ( + /\b(?:mkdir|touch|rm|mv|cp|install|tee)\b/.test(command) || + /\b(?:cat|echo|printf)\b.*(?:>|>>|\|\s*tee\b)/.test(command) || + /\bsed\s+-i\b/.test(command) || + /\bperl\s+-pi\b/.test(command) || + // In command position only: `cat AGENTS.wmill.md` is a read. + /(?:^|[;&|(]|\b(?:then|do|xargs)\s)\s*wmill(?:\s|$)/m.test(command) + ); +} diff --git a/ai_evals/adapters/cli/runtime.test.ts b/ai_evals/adapters/cli/runtime.test.ts index 3ea53ca587..d3e537b659 100644 --- a/ai_evals/adapters/cli/runtime.test.ts +++ b/ai_evals/adapters/cli/runtime.test.ts @@ -2,7 +2,9 @@ import { describe, expect, it } from "bun:test"; import { anthropicUsageToBenchmarkTokenUsage, extractCliResultTokenUsage, + endpointEnv, extractProposedWmillCommands, + getFirstMutationToolIndex, parseWmillInvocationLog, } from "./runtime"; @@ -147,3 +149,32 @@ push ]); }); }); + +describe("getFirstMutationToolIndex", () => { + it("counts wmill only as a command, not in a file name", () => { + const bash = (command: string) => ({ tool: "Bash", input: { command }, timestamp: 0 }); + expect( + getFirstMutationToolIndex([ + bash("cat AGENTS.wmill.md .agents/skills/x/SKILL.md"), + bash("ls && wmill generate-metadata"), + ]) + ).toBe(1); + expect(getFirstMutationToolIndex([bash("cd f\nwmill sync push")])).toBe(0); + }); +}); + +describe("endpointEnv", () => { + it("passes the endpoint only its own key", () => { + const env = endpointEnv( + { baseUrl: "https://example.test", apiKeyEnv: "DEEPSEEK_API_KEY" }, + "m", + { PATH: "/bin", DEEPSEEK_API_KEY: "ds", OPENAI_API_KEY: "oa", ANTHROPIC_AUTH_TOKEN: "t" } + ); + expect(env).toEqual({ + PATH: "/bin", + ANTHROPIC_BASE_URL: "https://example.test", + ANTHROPIC_API_KEY: "ds", + ANTHROPIC_DEFAULT_HAIKU_MODEL: "m", + }); + }); +}); diff --git a/ai_evals/adapters/cli/runtime.ts b/ai_evals/adapters/cli/runtime.ts index b45df70011..e7368db366 100644 --- a/ai_evals/adapters/cli/runtime.ts +++ b/ai_evals/adapters/cli/runtime.ts @@ -2,7 +2,14 @@ import { query, type Options } from "@anthropic-ai/claude-agent-sdk"; import { chmod, mkdir, readFile, writeFile } from "node:fs/promises"; import { delimiter, join } from "path"; import { fileURLToPath } from "url"; -import { getCliEvalModel, resolveEvalModel, type CliEvalModelConfig } from "../../core/models"; +import { + getCliEvalModel, + resolveEvalModel, + type CliEvalModelConfig, +} from "../../core/models"; +import { runCodex, type AgentRunResult } from "./codex"; +import { isLikelyMutatingBashCommand } from "./mutation"; +export { formatCliRunModelLabel } from "../../core/models"; import type { BenchmarkTokenUsage, CliToolInvocation, @@ -141,16 +148,94 @@ export async function runPromptAndCapture( maxTurns: number = 3, modelConfig: CliEvalModelConfig = DEFAULT_CLI_EVAL_MODEL ): Promise { - const toolsUsed: ToolInvocation[] = []; - const skillsInvoked: string[] = []; - const bashCommands: string[] = []; - let output = ""; - let assistantMessageCount = 0; - let tokenUsage: BenchmarkTokenUsage | null = null; - let finalContextTokens: number | null = null; const startedAt = Date.now(); const stubBinDir = join(cwd, WMILL_STUB_DIR_NAME); const wmillLogPath = join(cwd, WMILL_LOG_FILE_NAME); + const env = { + ...getQueryEnv(), + PATH: process.env.PATH ? `${stubBinDir}${delimiter}${process.env.PATH}` : stubBinDir, + WMILL_BENCHMARK_LOG_PATH: wmillLogPath, + }; + + await installWmillStub(stubBinDir); + + const run = + modelConfig.runtime === "codex" + ? await runCodex(prompt, cwd, maxTurns, modelConfig.model, env) + : await runClaudeCode(prompt, cwd, maxTurns, modelConfig, env); + + const skillsInvoked: string[] = []; + const bashCommands: string[] = []; + for (const tool of run.toolsUsed) { + if (tool.tool === "Skill" && typeof tool.input.skill === "string") { + pushUnique(skillsInvoked, tool.input.skill); + } + if (tool.tool === "Bash") { + for (const command of extractBashCommands(tool.input)) { + pushUnique(bashCommands, command); + } + } + } + + const wmillInvocations = await readWmillInvocationLog(wmillLogPath); + + return { + output: run.output, + durationMs: Date.now() - startedAt, + tokenUsage: run.tokenUsage, + finalContextTokens: run.finalContextTokens, + trace: { + toolsUsed: run.toolsUsed, + skillsInvoked, + assistantMessageCount: run.assistantMessageCount, + bashCommands, + proposedCommands: extractProposedWmillCommands(run.output), + executedWmillCommands: wmillInvocations.map(formatExecutedWmillCommand), + wmillInvocations, + firstMutationToolIndex: getFirstMutationToolIndex(run.toolsUsed), + }, + }; +} + +// Only the endpoint's own key may reach it: a missing key would make Claude Code +// fall back to the user's Claude login, and the agent's Bash inherits this env, +// so any other credential could end up in a tool result sent to the endpoint. +export function endpointEnv( + endpoint: NonNullable, + model: string, + env: Record +): Record { + const key = env[endpoint.apiKeyEnv]; + if (!key) { + throw new Error(`${endpoint.apiKeyEnv} is not set`); + } + const rest = Object.fromEntries( + Object.entries(env).filter(([name]) => !/KEY|TOKEN|SECRET|PASSWORD|CREDENTIAL/i.test(name)) + ); + return { + ...rest, + ANTHROPIC_BASE_URL: endpoint.baseUrl, + ANTHROPIC_API_KEY: key, + // Claude Code's background calls default to Haiku, which the endpoint does not serve. + ANTHROPIC_DEFAULT_HAIKU_MODEL: model, + }; +} + +async function runClaudeCode( + prompt: string, + cwd: string, + maxTurns: number, + modelConfig: CliEvalModelConfig, + env: Record +): Promise { + const result: AgentRunResult = { + output: "", + toolsUsed: [], + assistantMessageCount: 0, + tokenUsage: null, + finalContextTokens: null, + }; + const endpoint = modelConfig.anthropicEndpoint; const options: Options = { cwd, @@ -158,80 +243,42 @@ export async function runPromptAndCapture( maxTurns, settingSources: ["project"], allowedTools: ["Skill", "Read", "Glob", "Grep", "Bash", "Write", "Edit"], - env: { - ...getQueryEnv(), - PATH: process.env.PATH ? `${stubBinDir}${delimiter}${process.env.PATH}` : stubBinDir, - WMILL_BENCHMARK_LOG_PATH: wmillLogPath, - }, + env: endpoint ? endpointEnv(endpoint, modelConfig.model, env) : env, }; - await installWmillStub(stubBinDir); - for await (const message of query({ prompt, options })) { if (message.type === "assistant") { - assistantMessageCount += 1; + result.assistantMessageCount += 1; const turnContext = anthropicUsageToBenchmarkTokenUsage( message.message?.usage )?.prompt; if (turnContext && turnContext > 0) { - finalContextTokens = turnContext; + result.finalContextTokens = turnContext; } const content = message.message?.content; if (Array.isArray(content)) { for (const block of content) { if (block.type === "tool_use") { - const input = normalizeToolInput(block.input); - toolsUsed.push({ + result.toolsUsed.push({ tool: block.name, - input, + input: normalizeToolInput(block.input), timestamp: Date.now() }); - - if (block.name === "Skill") { - const skillInput = input as { skill?: string }; - if (skillInput.skill) { - pushUnique(skillsInvoked, skillInput.skill); - } - } - - if (block.name === "Bash") { - for (const command of extractBashCommands(input)) { - pushUnique(bashCommands, command); - } - } } else if (block.type === "text") { - output += block.text; + result.output += block.text; } } } } else if (message.type === "result") { const resultMessage = message as { result?: string }; - tokenUsage = extractCliResultTokenUsage(message) ?? tokenUsage; + result.tokenUsage = extractCliResultTokenUsage(message) ?? result.tokenUsage; if (typeof resultMessage.result === "string") { - output += resultMessage.result; + result.output += resultMessage.result; } } } - const proposedCommands = extractProposedWmillCommands(output); - const wmillInvocations = await readWmillInvocationLog(wmillLogPath); - - return { - output, - durationMs: Date.now() - startedAt, - tokenUsage, - finalContextTokens, - trace: { - toolsUsed, - skillsInvoked, - assistantMessageCount, - bashCommands, - proposedCommands, - executedWmillCommands: wmillInvocations.map(formatExecutedWmillCommand), - wmillInvocations, - firstMutationToolIndex: getFirstMutationToolIndex(toolsUsed), - }, - }; + return result; } export function wasSkillInvoked(result: PromptRunResult, skillName: string): boolean { @@ -242,10 +289,6 @@ export function wasToolUsed(result: PromptRunResult, toolName: string): boolean return result.trace.toolsUsed.some((tool) => tool.tool === toolName); } -export function formatCliRunModelLabel(modelConfig: CliEvalModelConfig): string { - return `${modelConfig.provider}:${modelConfig.model}`; -} - export function getToolInputs( result: PromptRunResult, toolName: string @@ -512,7 +555,7 @@ function formatExecutedWmillCommand(entry: CliWmillInvocation): string { return ["wmill", ...entry.argv].join(" ").trim(); } -function getFirstMutationToolIndex(toolsUsed: ToolInvocation[]): number | null { +export function getFirstMutationToolIndex(toolsUsed: ToolInvocation[]): number | null { for (const [index, tool] of toolsUsed.entries()) { if (tool.tool === "Write" || tool.tool === "Edit") { return index; @@ -526,16 +569,6 @@ function getFirstMutationToolIndex(toolsUsed: ToolInvocation[]): number | null { return null; } -function isLikelyMutatingBashCommand(command: string): boolean { - return ( - /\b(?:mkdir|touch|rm|mv|cp|install|tee)\b/.test(command) || - /\b(?:cat|echo|printf)\b.*(?:>|>>|\|\s*tee\b)/.test(command) || - /\bsed\s+-i\b/.test(command) || - /\bperl\s+-pi\b/.test(command) || - /\bwmill\b/.test(command) - ); -} - function pushUnique(values: string[], value: string): void { if (!values.includes(value)) { values.push(value); diff --git a/ai_evals/adapters/frontend/core/global/globalEvalRunner.ts b/ai_evals/adapters/frontend/core/global/globalEvalRunner.ts index be8c1d74cc..fe20d77d05 100644 --- a/ai_evals/adapters/frontend/core/global/globalEvalRunner.ts +++ b/ai_evals/adapters/frontend/core/global/globalEvalRunner.ts @@ -123,6 +123,7 @@ export interface GlobalEvalOptions { planMode?: boolean; model?: string; maxIterations?: number; + userAnswers?: string[]; provider?: AIProvider; backend: WindmillBackendSettings; workspaceRoot?: string; @@ -220,6 +221,7 @@ export async function runGlobalEval( onToolCall: options.runContext?.onToolCall, options: { maxIterations: options.maxIterations, + userAnswers: options.userAnswers, model, workspace: workspaceRoot, provider: options.provider, diff --git a/ai_evals/adapters/frontend/core/shared/baseEvalRunner.ts b/ai_evals/adapters/frontend/core/shared/baseEvalRunner.ts index 3d0c6d6734..13aaa59806 100644 --- a/ai_evals/adapters/frontend/core/shared/baseEvalRunner.ts +++ b/ai_evals/adapters/frontend/core/shared/baseEvalRunner.ts @@ -85,6 +85,7 @@ export async function runEval( const model = options.model ?? "gpt-4o"; const maxIterations = options.maxIterations ?? 20; + const pendingAnswers = options.userAnswers && [...options.userAnswers]; const workspace = options.workspace ?? "test-workspace"; const provider = toFrontendEvalProvider(options.provider); @@ -138,6 +139,12 @@ export async function runEval( // edit the arguments, so a case can assert what the model proposed but never how // it reacts to the user changing something. requestRunArgs: async (_toolId, form) => form.args, + ...(pendingAnswers && { + requestUserQuestion: async () => { + const answer = pendingAnswers.shift(); + return answer === undefined ? undefined : [answer]; + }, + }), onNewToken: (token: string) => { if (shouldEmitMessageStart) { onAssistantMessageStart?.(); diff --git a/ai_evals/adapters/frontend/core/shared/types.ts b/ai_evals/adapters/frontend/core/shared/types.ts index c7af6c43e7..1ff783e258 100644 --- a/ai_evals/adapters/frontend/core/shared/types.ts +++ b/ai_evals/adapters/frontend/core/shared/types.ts @@ -16,6 +16,9 @@ export interface ToolCallDetail { export interface EvalRunnerOptions { backend: WindmillBackendSettings; maxIterations?: number; + // Scripted answers to the model's questions, one per question in order; a question past + // the last one is dismissed. Unset, the run cannot be asked anything. + userAnswers?: string[]; model?: string; workspace?: string; provider?: AIProvider; diff --git a/ai_evals/adapters/frontend/vitestAdapter.test.ts b/ai_evals/adapters/frontend/vitestAdapter.test.ts index 9adb4d05ac..288c67e3eb 100644 --- a/ai_evals/adapters/frontend/vitestAdapter.test.ts +++ b/ai_evals/adapters/frontend/vitestAdapter.test.ts @@ -6,6 +6,8 @@ import { dirname, resolve } from 'path' // @ts-ignore - Node.js url import { fileURLToPath } from 'url' import { handleBenchmarkApiFetch, hasBenchmarkApiHandler } from './mockBackend' +import { resolveWindmillBackendSettings } from '../../core/windmillBackendSettings' +import { WindmillBackendClient } from './windmillBackend' const FRONTEND_DIR = fileURLToPath(new URL('../../../frontend/', import.meta.url)) @@ -28,6 +30,11 @@ globalThis.fetch = (async (input: unknown, init?: RequestInit) => { if (typeof url === 'string' && hasBenchmarkApiHandler(url)) { return handleBenchmarkApiFetch(url, init) } + // The docs tools read the documentation bundled into the backend: hand them to the + // benchmark's real backend, logged in like the AI proxy calls. + if (typeof url === 'string' && url.startsWith('/api/docs/')) { + return new WindmillBackendClient(resolveWindmillBackendSettings()).request(url.slice('/api'.length), init) + } // The parsers behind inferArgs load their wasm from a vite `?url` path, which only a dev // server serves. Unserved, every script schema infers as empty, and the tools then tell the // model its arguments are undeclared, so it rewrites a correct script until it runs out of turns. @@ -684,6 +691,7 @@ benchmarkIt( resetBenchmarkMockBackend() } }, - // Full-suite runs (30+ cases at concurrency 2-3) routinely exceed 10 minutes. - 7_200_000 + // A full suite on a slow reasoning model runs for hours (gpt-6-astra on global went past + // 2h); the CI job's own timeout is the real bound. + 18_000_000 ) diff --git a/ai_evals/cases/global.yaml b/ai_evals/cases/global.yaml index 476218a066..b66af3e8dd 100644 --- a/ai_evals/cases/global.yaml +++ b/ai_evals/cases/global.yaml @@ -198,22 +198,127 @@ - uses the requested placeholder value and description - does not create a resource or deploy anything +# A new app that has to store data, with nothing said about where: the data setup is the +# user's call, so it is asked before anything is created. The question is dismissed here, +# which leaves nothing the model may build on. - id: global-test7-ambiguous-app-asks-question prompt: |- Create a new raw app for triaging support tickets. + initial: ai_evals/fixtures/frontend/global/initial/datatable_orders_seed.json runtime: - maxTurns: 4 + maxTurns: 6 + userAnswers: [] validate: draftCountExactly: 0 toolExpect: requiredToolsUsed: + - list_datatables - askUserQuestion forbiddenToolsUsed: - init_app + - exec_datatable_sql - write_app_file - write_app_runnable - deploy_workspace_item - delete_workspace_item + toolCallArgs: + - tool: askUserQuestion + field: question + stringIncludesAnyOf: + - data table + - datatable + - schema + - tables + - store + skipJudge: true + +# The same request with the question answered: the app is built on what the user chose. +- id: global-new-app-data-setup-answer-applied + prompt: |- + Create a new raw app for triaging support tickets. + initial: ai_evals/fixtures/frontend/global/initial/datatable_orders_seed.json + runtime: + maxTurns: 32 + userAnswers: + - A new schema named helpdesk + - Create new tables + validate: + draftCountExactly: 1 + requiredDrafts: + - type: app + valueIncludes: + - helpdesk + - react-dom/client + toolExpect: + requiredToolsUsed: + - askUserQuestion + - init_app + forbiddenToolsUsed: + - deploy_workspace_item + toolCallArgs: + - tool: init_app + field: framework + stringEqualsAnyOf: + - react19 + - tool: exec_datatable_sql + field: sql + stringIncludesAnyOf: + - helpdesk + skipJudge: true + +# No framework named, no app to follow, nothing to store: the modal's recommended +# framework is used without a question. +- id: global-new-app-defaults-to-react19 + prompt: |- + Build me a little app that converts temperatures between Celsius and Fahrenheit. + runtime: + maxTurns: 12 + validate: + draftCountExactly: 1 + requiredDrafts: + - type: app + valueIncludes: + - react-dom/client + toolExpect: + requiredToolsUsed: + - init_app + forbiddenToolsUsed: + - askUserQuestion + - exec_datatable_sql + - deploy_workspace_item + toolCallArgs: + - tool: init_app + field: framework + stringEqualsAnyOf: + - react19 + skipJudge: true + +# The user said where the data goes, so there is nothing left to ask. +- id: global-new-app-storage-specified-no-question + prompt: |- + Build a todo list app in React. Keep the todos in the main data table, in a new + schema called todo_app. + initial: ai_evals/fixtures/frontend/global/initial/datatable_orders_seed.json + runtime: + maxTurns: 20 + validate: + draftCountExactly: 1 + requiredDrafts: + - type: app + valueIncludes: + - todo_app + toolExpect: + requiredToolsUsed: + - init_app + - write_app_runnable + forbiddenToolsUsed: + - askUserQuestion + - deploy_workspace_item + toolCallArgs: + - tool: exec_datatable_sql + field: sql + stringIncludesAnyOf: + - todo_app skipJudge: true - id: global-test8-human-script-infer-path-language diff --git a/ai_evals/core/cases.test.ts b/ai_evals/core/cases.test.ts index 5d6e3245db..b01d4143ba 100644 --- a/ai_evals/core/cases.test.ts +++ b/ai_evals/core/cases.test.ts @@ -192,8 +192,6 @@ describe("loadCases", () => { requiredSkills: ["write-script-bun"], requiredSkillsBeforeFirstMutation: ["write-script-bun"], forbiddenSkills: ["write-script-python3", "write-flow"], - orderedAssistantMentions: ["wmill generate-metadata", "wmill sync push"], - orderedProposedCommands: ["wmill generate-metadata", "wmill sync push"], forbiddenExecutedCommands: ["^wmill generate-metadata", "^wmill sync push"], }); }); diff --git a/ai_evals/core/models.test.ts b/ai_evals/core/models.test.ts index d51d61d224..96f16778d0 100644 --- a/ai_evals/core/models.test.ts +++ b/ai_evals/core/models.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from "bun:test"; -import { resolveEvalModel } from "./models"; +import { formatRunModelLabel, resolveEvalModel } from "./models"; describe("resolveEvalModel", () => { it("supports GPT-5.5 aliases for frontend evals", () => { @@ -47,6 +47,18 @@ describe("resolveEvalModel", () => { }); }); + it("labels cli runs by agent, keeping Claude's historical label", () => { + expect(formatRunModelLabel("cli", resolveEvalModel("cli", "sonnet-5.5"))).toBe( + "anthropic:claude-sonnet-5-5", + ); + expect(formatRunModelLabel("cli", resolveEvalModel("cli", "gpt-6.1-sol"))).toBe( + "codex:gpt-6.1-sol", + ); + expect(formatRunModelLabel("cli", resolveEvalModel("cli", "deepseek-v4-pro"))).toBe( + "claude-code:deepseek-v4-pro", + ); + }); + it("rejects Gemini aliases for cli evals", () => { expect(() => resolveEvalModel("cli", "gemini-3-flash-preview")).toThrow( "Model gemini-3-flash-preview is not supported for cli mode", diff --git a/ai_evals/core/models.ts b/ai_evals/core/models.ts index 06689c36e1..cf131337fd 100644 --- a/ai_evals/core/models.ts +++ b/ai_evals/core/models.ts @@ -5,9 +5,13 @@ export interface FrontendEvalModelConfig { model: string; } +// The coding agent that drives the model in cli mode (adapters/cli/runtime.ts). export interface CliEvalModelConfig { - provider: "anthropic"; + runtime: "claude-code" | "codex"; model: string; + // An Anthropic-compatible endpoint other than Anthropic's, for claude-code, + // and the env var holding its key. + anthropicEndpoint?: { baseUrl: string; apiKeyEnv: string }; } export interface EvalModelSpec { @@ -35,7 +39,7 @@ export const EVAL_MODELS: EvalModelSpec[] = [ model: "claude-haiku-4-5-20251001", }, cli: { - provider: "anthropic", + runtime: "claude-code", model: "haiku", }, }, @@ -55,7 +59,7 @@ export const EVAL_MODELS: EvalModelSpec[] = [ model: "claude-sonnet-4-5-20250929", }, cli: { - provider: "anthropic", + runtime: "claude-code", model: "sonnet", }, }, @@ -74,7 +78,7 @@ export const EVAL_MODELS: EvalModelSpec[] = [ model: "claude-opus-4-6", }, cli: { - provider: "anthropic", + runtime: "claude-code", model: "opus", }, }, @@ -87,7 +91,7 @@ export const EVAL_MODELS: EvalModelSpec[] = [ model: "claude-sonnet-5-5", }, cli: { - provider: "anthropic", + runtime: "claude-code", model: "claude-sonnet-5-5", }, }, @@ -100,7 +104,7 @@ export const EVAL_MODELS: EvalModelSpec[] = [ model: "claude-opus-5-5", }, cli: { - provider: "anthropic", + runtime: "claude-code", model: "claude-opus-5-5", }, }, @@ -139,6 +143,19 @@ export const EVAL_MODELS: EvalModelSpec[] = [ provider: "openai", model: "gpt-6-astra", }, + cli: { + runtime: "codex", + model: "gpt-6-astra", + }, + }, + { + id: "gpt-6.1-sol", + label: "GPT-6.1 Sol", + aliases: ["gpt-6.1-sol"], + frontend: { + provider: "openai", + model: "gpt-6.1-sol", + }, }, { id: "gpt-6-sol", @@ -149,6 +166,19 @@ export const EVAL_MODELS: EvalModelSpec[] = [ model: "gpt-6-sol", }, }, + { + id: "gpt-6.1-sol", + label: "GPT-6.1 Sol", + aliases: ["gpt-6.1-sol"], + frontend: { + provider: "openai", + model: "gpt-6.1-sol", + }, + cli: { + runtime: "codex", + model: "gpt-6.1-sol", + }, + }, { id: "gpt-6-luna", label: "GPT-6 Luna", @@ -215,6 +245,14 @@ export const EVAL_MODELS: EvalModelSpec[] = [ provider: "deepseek", model: "deepseek-v4-pro", }, + cli: { + runtime: "claude-code", + model: "deepseek-v4-pro", + anthropicEndpoint: { + baseUrl: "https://api.deepseek.com/anthropic", + apiKeyEnv: "DEEPSEEK_API_KEY", + }, + }, }, ]; @@ -253,11 +291,20 @@ export function formatRunModelLabel( model: EvalModelSpec, ): string { if (mode === "cli") { - return `${model.cli!.provider}:${model.cli!.model}`; + return formatCliRunModelLabel(model.cli!); } return `${model.frontend!.provider}:${model.frontend!.model}`; } +// Claude on Claude Code keeps the `anthropic:` prefix its dashboard history was +// recorded under; any other pairing names the agent. +export function formatCliRunModelLabel(config: CliEvalModelConfig): string { + if (config.runtime === "claude-code" && !config.anthropicEndpoint) { + return `anthropic:${config.model}`; + } + return `${config.runtime}:${config.model}`; +} + export function getFrontendEvalModel( model: EvalModelSpec, ): FrontendEvalModelConfig { diff --git a/ai_evals/core/types.ts b/ai_evals/core/types.ts index 832a383a73..7e70191992 100644 --- a/ai_evals/core/types.ts +++ b/ai_evals/core/types.ts @@ -29,6 +29,10 @@ export interface EvalCaseRuntimeAppContextSpec { export interface EvalCaseRuntimeSpec { maxTurns?: number; + // Global mode: scripted answers to askUserQuestion, one per question in order; a question + // past the last one is dismissed, so `[]` dismisses them all. Unset, the question tool + // reports that it cannot ask. + userAnswers?: string[]; backendPreview?: EvalCaseRuntimeBackendPreview; appContext?: EvalCaseRuntimeAppContextSpec; // Global mode: run as a session chat (preview tools + session prompt) vs the standalone chat. diff --git a/ai_evals/modes/cli.ts b/ai_evals/modes/cli.ts index 205cdfa9c4..f337f99f40 100644 --- a/ai_evals/modes/cli.ts +++ b/ai_evals/modes/cli.ts @@ -39,17 +39,36 @@ interface CliRunActual { trace: CliTrace; } -const CLAUDE_PROJECT_PREAMBLE = [ - "Follow the project instructions from AGENTS.md exactly.", - "Before creating or modifying any Windmill entity, you MUST invoke the relevant Skill tool and follow it.", - "Use the skill guidance for file layout, implementation details, and the exact next commands to tell the user.", - "Do not skip the Skill step.", +// Only the skill wording differs per agent: Claude Code has a Skill tool, Codex +// loads a skill by reading its SKILL.md. +const SKILL_STEP: Record = { + "claude-code": [ + "Before creating or modifying any Windmill entity, you MUST invoke the relevant Skill tool and follow it.", + "Use the skill guidance for file layout, implementation details, and the exact next commands to tell the user.", + "Do not skip the Skill step.", + ], + codex: [ + "Before creating or modifying any Windmill entity, you MUST read the relevant skill's SKILL.md under .agents/skills and follow it.", + "Use the skill guidance for file layout, implementation details, and the exact next commands to tell the user.", + "Do not skip the skill step.", + ], +}; + +const HARNESS_PREAMBLE = [ "You are running inside an automated benchmark harness, not an interactive user session.", "Act autonomously and complete the requested file changes directly in the workspace.", "Do not ask for confirmation, do not ask the user to save or create files manually, and do not wait for approval.", "Do not respond with a plan when you can make the change directly.", "Only describe what was done after you have written the files.", -].join(" "); +]; + +function projectPreamble(runtime: CliEvalModelConfig["runtime"]): string { + return [ + "Follow the project instructions from AGENTS.md exactly.", + ...SKILL_STEP[runtime], + ...HARNESS_PREAMBLE, + ].join(" "); +} export function createCliModeRunner( modelConfig: CliEvalModelConfig = DEFAULT_CLI_EVAL_MODEL @@ -89,7 +108,7 @@ export function createCliModeRunner( }); await writeFile(join(workspaceDir, "rt.d.ts"), "export namespace RT {}\n", "utf8"); - const renderedPrompt = await renderPrompt(prompt, workspaceDir); + const renderedPrompt = await renderPrompt(prompt, workspaceDir, modelConfig.runtime); const run = await runPromptAndCapture( renderedPrompt, workspaceDir, @@ -193,7 +212,11 @@ export function getCliRunModelLabel( return formatCliRunModelLabel(modelConfig); } -async function renderPrompt(prompt: string, workspaceDir: string): Promise { +async function renderPrompt( + prompt: string, + workspaceDir: string, + runtime: CliEvalModelConfig["runtime"] +): Promise { const renderedUserPrompt = prompt.replaceAll("{{workspace_root}}", workspaceDir); const agentsInstructions = await readFile(path.join(workspaceDir, "AGENTS.md"), "utf8"); @@ -202,7 +225,7 @@ async function renderPrompt(prompt: string, workspaceDir: string): Promise', true)\n ", "describe": { "columns": [], "parameters": { @@ -10,5 +10,5 @@ }, "nullable": [] }, - "hash": "782c259ccdf71c5e2fa8dcb27515e7c5ff83009774dc15095ed0dbcb0ecdbdde" + "hash": "009fd85c1aa55ea36454eafffbf62505461eb1cc8b9af85a2e2ab6e7dbe950c6" } diff --git a/backend/.sqlx/query-03cc00b0b5b8b1ffcc14ac5f6cb3062a8f706e70bc9c4b3764cd1d6f99b9440a.json b/backend/.sqlx/query-03cc00b0b5b8b1ffcc14ac5f6cb3062a8f706e70bc9c4b3764cd1d6f99b9440a.json new file mode 100644 index 0000000000..ccd8b05822 --- /dev/null +++ b/backend/.sqlx/query-03cc00b0b5b8b1ffcc14ac5f6cb3062a8f706e70bc9c4b3764cd1d6f99b9440a.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "\nINSERT INTO concurrency_counter(concurrency_id, job_uuids)\nVALUES ($1, '{}'::jsonb)\nON CONFLICT (concurrency_id)\nDO UPDATE SET job_uuids = concurrency_counter.job_uuids\nRETURNING (SELECT COUNT(*)\n FROM jsonb_object_keys(job_uuids) AS keys(key)\n WHERE key <> $2)", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "count", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Varchar", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "03cc00b0b5b8b1ffcc14ac5f6cb3062a8f706e70bc9c4b3764cd1d6f99b9440a" +} diff --git a/backend/.sqlx/query-05b610ad1809d487c32e85b0392cfca8913ba1129fdf4978a8a4738689786160.json b/backend/.sqlx/query-05b610ad1809d487c32e85b0392cfca8913ba1129fdf4978a8a4738689786160.json new file mode 100644 index 0000000000..5c6ee3d506 --- /dev/null +++ b/backend/.sqlx/query-05b610ad1809d487c32e85b0392cfca8913ba1129fdf4978a8a4738689786160.json @@ -0,0 +1,19 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO script\n (workspace_id, hash, path, parent_hashes, summary, description, content, created_by, schema, is_template, extra_perms, lock, language, kind, tag, envs, concurrent_limit, concurrency_time_window_s, cache_ttl, cache_ignore_s3_path, dedicated_worker, ws_error_handler_muted, priority, restart_unless_cancelled, delete_after_use, delete_after_secs, timeout, concurrency_key, visible_to_runner_only, auto_kind, codebase, has_preprocessor, on_behalf_of, on_behalf_of_email, schema_validation, assets, debounce_key, debounce_delay_s, runnable_settings_handle, modules, labels, job_token_scopes, lock_error_logs, created_at)\n\n SELECT workspace_id, $1, path, array_prepend($2::bigint, COALESCE(parent_hashes, '{}'::bigint[])), summary, description, content, created_by, schema, is_template, extra_perms, $4::text, language, kind, tag, envs, concurrent_limit, concurrency_time_window_s, cache_ttl, cache_ignore_s3_path, dedicated_worker, ws_error_handler_muted, priority, restart_unless_cancelled, delete_after_use, delete_after_secs, timeout, concurrency_key, visible_to_runner_only, auto_kind, codebase, has_preprocessor, on_behalf_of, on_behalf_of_email, schema_validation, assets, debounce_key, debounce_delay_s, runnable_settings_handle, COALESCE($5::jsonb, modules), labels, job_token_scopes, $6::text, clock_timestamp()\n\n FROM script WHERE hash = $2 AND workspace_id = $3;\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int8", + "Int8", + "Text", + "Text", + "Jsonb", + "Text" + ] + }, + "nullable": [] + }, + "hash": "05b610ad1809d487c32e85b0392cfca8913ba1129fdf4978a8a4738689786160" +} diff --git a/backend/.sqlx/query-095fa75c60724664f0355cd6e3a64f84a0f1a31cd6732c0ab994a2d6ac3c3eec.json b/backend/.sqlx/query-095fa75c60724664f0355cd6e3a64f84a0f1a31cd6732c0ab994a2d6ac3c3eec.json new file mode 100644 index 0000000000..852ce508ff --- /dev/null +++ b/backend/.sqlx/query-095fa75c60724664f0355cd6e3a64f84a0f1a31cd6732c0ab994a2d6ac3c3eec.json @@ -0,0 +1,152 @@ +{ + "db_name": "PostgreSQL", + "query": "WITH inserted_job AS (\n INSERT INTO v2_job (\n id, -- 1\n workspace_id, -- 2\n raw_code, -- 3\n raw_lock, -- 4\n raw_flow, -- 5\n tag, -- 6\n parent_job, -- 7\n created_by, -- 8\n permissioned_as, -- 9\n runnable_id, -- 10\n runnable_path, -- 11\n args, -- 12\n kind, -- 13\n trigger, -- 14\n script_lang, -- 15\n same_worker, -- 16\n pre_run_error, -- 17\n permissioned_as_email, -- 18\n visible_to_owner, -- 19\n flow_innermost_root_job, -- 20\n root_job, -- 38\n concurrent_limit, -- 21\n concurrency_time_window_s, -- 22\n timeout, -- 23\n flow_step_id, -- 24\n cache_ttl, -- 25\n priority, -- 26\n trigger_kind, -- 39\n script_entrypoint_override, -- 12\n preprocessed, -- 27,\n labels -- 44\n ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18,\n $19, $20, $38, $21, $22, $23, $24, $25, $26, $39::job_trigger_kind,\n ($12::JSONB)->>'_ENTRYPOINT_OVERRIDE', $27,\n -- $44 (payload labels) merged with the labels of the runnable's folder, if any\n -- ($45/$46 are runnable_path/workspace_id again, kept separate to avoid parameter type conflicts)\n (SELECT CASE WHEN fl.labels IS NULL THEN $44\n ELSE (SELECT array_agg(DISTINCT lbl) FROM unnest(COALESCE($44, ARRAY[]::TEXT[]) || fl.labels) lbl)\n END\n FROM folder_labels($46, $45) AS fl(labels)))\n ),\n inserted_runtime AS (\n INSERT INTO v2_job_runtime (id, ping) VALUES ($1, null)\n ),\n inserted_job_perms AS (\n INSERT INTO job_perms (job_id, email, username, is_admin, is_operator, folders, groups, workspace_id, end_user_email, job_token_scopes)\n values ($1, $32, $33, $34, $35, $36, $37, $2, $41, $47)\n ON CONFLICT (job_id) DO UPDATE SET email = EXCLUDED.email, username = EXCLUDED.username, is_admin = EXCLUDED.is_admin, is_operator = EXCLUDED.is_operator, folders = EXCLUDED.folders, groups = EXCLUDED.groups, workspace_id = EXCLUDED.workspace_id, end_user_email = EXCLUDED.end_user_email, job_token_scopes = EXCLUDED.job_token_scopes\n )\n INSERT INTO v2_job_queue\n (workspace_id, id, running, scheduled_for, started_at, tag, priority, cache_ignore_s3_path, runnable_settings_handle)\n VALUES ($2, $1, $28, COALESCE($29, now()), CASE WHEN $27 OR $40 THEN now() END, $30, $31, $42, $43)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Uuid", + "Varchar", + "Text", + "Text", + "Jsonb", + "Varchar", + "Uuid", + "Varchar", + "Varchar", + "Int8", + "Varchar", + "Jsonb", + { + "Custom": { + "name": "job_kind", + "kind": { + "Enum": [ + "script", + "preview", + "flow", + "dependencies", + "flowpreview", + "script_hub", + "identity", + "flowdependencies", + "http", + "graphql", + "postgresql", + "noop", + "appdependencies", + "deploymentcallback", + "singlestepflow", + "flowscript", + "flownode", + "appscript", + "aiagent", + "unassigned_script", + "unassigned_flow", + "unassigned_singlestepflow" + ] + } + } + }, + "Varchar", + { + "Custom": { + "name": "script_lang", + "kind": { + "Enum": [ + "python3", + "deno", + "go", + "bash", + "postgresql", + "nativets", + "bun", + "mysql", + "bigquery", + "snowflake", + "graphql", + "powershell", + "mssql", + "php", + "bunnative", + "rust", + "ansible", + "csharp", + "oracledb", + "nu", + "java", + "duckdb", + "ruby", + "rlang", + "dbt" + ] + } + } + }, + "Bool", + "Text", + "Varchar", + "Bool", + "Uuid", + "Int4", + "Int4", + "Int4", + "Varchar", + "Int4", + "Int2", + "Bool", + "Bool", + "Timestamptz", + "Varchar", + "Int2", + "Varchar", + "Varchar", + "Bool", + "Bool", + "JsonbArray", + "TextArray", + "Uuid", + { + "Custom": { + "name": "job_trigger_kind", + "kind": { + "Enum": [ + "webhook", + "http", + "websocket", + "kafka", + "email", + "nats", + "schedule", + "app", + "ui", + "postgres", + "sqs", + "gcp", + "mqtt", + "nextcloud", + "google", + "ci_test", + "github", + "azure", + "asset", + "freshness", + "amqp" + ] + } + } + }, + "Bool", + "Varchar", + "Bool", + "Int8", + "TextArray", + "Text", + "Text", + "TextArray" + ] + }, + "nullable": [] + }, + "hash": "095fa75c60724664f0355cd6e3a64f84a0f1a31cd6732c0ab994a2d6ac3c3eec" +} diff --git a/backend/.sqlx/query-0d260ba4f168d725ea3b82d33324dd1d16a68a2bae6e500d9e480b152fd5859d.json b/backend/.sqlx/query-0d260ba4f168d725ea3b82d33324dd1d16a68a2bae6e500d9e480b152fd5859d.json new file mode 100644 index 0000000000..28dfcf702f --- /dev/null +++ b/backend/.sqlx/query-0d260ba4f168d725ea3b82d33324dd1d16a68a2bae6e500d9e480b152fd5859d.json @@ -0,0 +1,47 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT DISTINCT ON (path)\n path,\n value as \"value!: sqlx::types::Json>\",\n created_at,\n email IS NULL as \"legacy!\",\n typ::text as \"typ!\"\n FROM draft\n WHERE workspace_id = $1\n AND typ IN ('app', 'raw_app')\n AND (email = $2 OR email IS NULL)\n AND NOT EXISTS (\n SELECT 1 FROM app a\n WHERE a.workspace_id = draft.workspace_id\n AND a.path = draft.path\n )\n ORDER BY path, (email IS NULL), created_at DESC", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "path", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "value!: sqlx::types::Json>", + "type_info": "Json" + }, + { + "ordinal": 2, + "name": "created_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 3, + "name": "legacy!", + "type_info": "Bool" + }, + { + "ordinal": 4, + "name": "typ!", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false, + false, + false, + null, + null + ] + }, + "hash": "0d260ba4f168d725ea3b82d33324dd1d16a68a2bae6e500d9e480b152fd5859d" +} diff --git a/backend/.sqlx/query-dbc05cbf6720cc85e01118bcdcbb478724ac0ea8f3e043da2de9d932216cc730.json b/backend/.sqlx/query-1043947475822c27f73b5357e6ec0d527d999f2865ff5de75186b49004a3565e.json similarity index 62% rename from backend/.sqlx/query-dbc05cbf6720cc85e01118bcdcbb478724ac0ea8f3e043da2de9d932216cc730.json rename to backend/.sqlx/query-1043947475822c27f73b5357e6ec0d527d999f2865ff5de75186b49004a3565e.json index 6e5f8645bf..a0847ae2ca 100644 --- a/backend/.sqlx/query-dbc05cbf6720cc85e01118bcdcbb478724ac0ea8f3e043da2de9d932216cc730.json +++ b/backend/.sqlx/query-1043947475822c27f73b5357e6ec0d527d999f2865ff5de75186b49004a3565e.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by, schema, summary, description, lock)\n VALUES ('test-workspace', $1, 'f/test/simple_script', $2, 'deno', 'script', 'test-user', '{}', 'Simple test script', 'A simple test script', '')\n ", + "query": "\n INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by, schema, summary, description, lock)\n VALUES ('test-workspace', $1, 'f/test/simple_script', $2, 'bun', 'script', 'test-user', '{}', 'Simple test script', 'A simple test script', E'{}\\n//bun.lock\\n')\n ", "describe": { "columns": [], "parameters": { @@ -11,5 +11,5 @@ }, "nullable": [] }, - "hash": "dbc05cbf6720cc85e01118bcdcbb478724ac0ea8f3e043da2de9d932216cc730" + "hash": "1043947475822c27f73b5357e6ec0d527d999f2865ff5de75186b49004a3565e" } diff --git a/backend/.sqlx/query-10f166464a10fc35df2b6ff1b497b17d6bb71f0ae69cd218e6a3fbc151d19d87.json b/backend/.sqlx/query-10f166464a10fc35df2b6ff1b497b17d6bb71f0ae69cd218e6a3fbc151d19d87.json new file mode 100644 index 0000000000..027f01d181 --- /dev/null +++ b/backend/.sqlx/query-10f166464a10fc35df2b6ff1b497b17d6bb71f0ae69cd218e6a3fbc151d19d87.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT EXISTS (\n SELECT 1 FROM v2_job o, v2_job t,\n LATERAL (SELECT ARRAY_REMOVE(ARRAY[o.id, o.parent_job, o.root_job,\n o.flow_innermost_root_job], NULL) AS run) l\n WHERE o.id = $1 AND t.id = $2 AND t.workspace_id = o.workspace_id\n AND (t.id = ANY(l.run) OR t.parent_job = ANY(l.run)\n OR t.root_job = ANY(l.run) OR t.flow_innermost_root_job = ANY(l.run))\n )", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "exists", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Uuid" + ] + }, + "nullable": [ + null + ] + }, + "hash": "10f166464a10fc35df2b6ff1b497b17d6bb71f0ae69cd218e6a3fbc151d19d87" +} diff --git a/backend/.sqlx/query-1333436e3d0466eccdaf99fb70aadad56b50619036fab3f2c1bdbbf280a80217.json b/backend/.sqlx/query-1333436e3d0466eccdaf99fb70aadad56b50619036fab3f2c1bdbbf280a80217.json new file mode 100644 index 0000000000..1946f55366 --- /dev/null +++ b/backend/.sqlx/query-1333436e3d0466eccdaf99fb70aadad56b50619036fab3f2c1bdbbf280a80217.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "WITH prev AS (SELECT started_at FROM v2_job_queue WHERE id = $1)\n UPDATE v2_job_queue q SET running = false, started_at = null,\n extras = CASE WHEN q.started_at IS NULL OR q.extras ? 'wac_first_started_at' THEN q.extras\n ELSE jsonb_set(COALESCE(q.extras, '{}'::jsonb), '{wac_first_started_at}', to_jsonb(q.started_at))\n END\n FROM prev WHERE q.id = $1\n RETURNING (extract(epoch FROM now() - prev.started_at) * 1000)::bigint", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "int8", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Uuid" + ] + }, + "nullable": [ + null + ] + }, + "hash": "1333436e3d0466eccdaf99fb70aadad56b50619036fab3f2c1bdbbf280a80217" +} diff --git a/backend/.sqlx/query-13f358b7c76cda0b24fcb17dbbe10a00333bbf3b8b43bbccc7a6e1926d82478c.json b/backend/.sqlx/query-13f358b7c76cda0b24fcb17dbbe10a00333bbf3b8b43bbccc7a6e1926d82478c.json new file mode 100644 index 0000000000..25aedfe3be --- /dev/null +++ b/backend/.sqlx/query-13f358b7c76cda0b24fcb17dbbe10a00333bbf3b8b43bbccc7a6e1926d82478c.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT job_token_scopes FROM job_perms WHERE job_id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "job_token_scopes", + "type_info": "TextArray" + } + ], + "parameters": { + "Left": [ + "Uuid" + ] + }, + "nullable": [ + true + ] + }, + "hash": "13f358b7c76cda0b24fcb17dbbe10a00333bbf3b8b43bbccc7a6e1926d82478c" +} diff --git a/backend/.sqlx/query-1ae4aab40783b4cd588e5a4da57eabca765789cd706ad682e8094c8960bc46a8.json b/backend/.sqlx/query-1ae4aab40783b4cd588e5a4da57eabca765789cd706ad682e8094c8960bc46a8.json new file mode 100644 index 0000000000..6ae002d4f7 --- /dev/null +++ b/backend/.sqlx/query-1ae4aab40783b4cd588e5a4da57eabca765789cd706ad682e8094c8960bc46a8.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM runnable_job_stats s WHERE s.hour >= $1 AND NOT EXISTS (SELECT 1 FROM workspace w WHERE w.id = s.workspace_id)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int8" + ] + }, + "nullable": [] + }, + "hash": "1ae4aab40783b4cd588e5a4da57eabca765789cd706ad682e8094c8960bc46a8" +} diff --git a/backend/.sqlx/query-1e3407ad2ec46bd965f2aeede0d3cd4a3f9226a5e0c6b15c2580ddd1dba6199b.json b/backend/.sqlx/query-1e3407ad2ec46bd965f2aeede0d3cd4a3f9226a5e0c6b15c2580ddd1dba6199b.json new file mode 100644 index 0000000000..036610e08c --- /dev/null +++ b/backend/.sqlx/query-1e3407ad2ec46bd965f2aeede0d3cd4a3f9226a5e0c6b15c2580ddd1dba6199b.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT wp.draining FROM v2_job_queue q JOIN worker_ping wp ON wp.worker = q.worker\n WHERE q.id = $1 AND q.workspace_id = $2 AND q.running AND q.started_at IS NOT NULL", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "draining", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "1e3407ad2ec46bd965f2aeede0d3cd4a3f9226a5e0c6b15c2580ddd1dba6199b" +} diff --git a/backend/.sqlx/query-1fd3d5e8918143c6e4828b092c5d7d376555217b037d7694efa5ceffc4980f80.json b/backend/.sqlx/query-1fd3d5e8918143c6e4828b092c5d7d376555217b037d7694efa5ceffc4980f80.json new file mode 100644 index 0000000000..988ff44bb4 --- /dev/null +++ b/backend/.sqlx/query-1fd3d5e8918143c6e4828b092c5d7d376555217b037d7694efa5ceffc4980f80.json @@ -0,0 +1,40 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT COALESCE((operator_settings->>'builder_flows')::boolean, false) AS \"flows!\",\n COALESCE((operator_settings->>'builder_apps')::boolean, false) AS \"apps!\",\n COALESCE((operator_settings->>'manage_schedules')::boolean, true) AS \"schedules!\",\n COALESCE((operator_settings->>'manage_triggers')::boolean, true) AS \"triggers!\"\n FROM workspace_settings WHERE workspace_id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "flows!", + "type_info": "Bool" + }, + { + "ordinal": 1, + "name": "apps!", + "type_info": "Bool" + }, + { + "ordinal": 2, + "name": "schedules!", + "type_info": "Bool" + }, + { + "ordinal": 3, + "name": "triggers!", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + null, + null, + null, + null + ] + }, + "hash": "1fd3d5e8918143c6e4828b092c5d7d376555217b037d7694efa5ceffc4980f80" +} diff --git a/backend/.sqlx/query-212bf5b32de102a9c537907aaff04befc3a6596c258086805dd42f513ddb3ead.json b/backend/.sqlx/query-212bf5b32de102a9c537907aaff04befc3a6596c258086805dd42f513ddb3ead.json deleted file mode 100644 index 04b23832d9..0000000000 --- a/backend/.sqlx/query-212bf5b32de102a9c537907aaff04befc3a6596c258086805dd42f513ddb3ead.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "INSERT INTO resource_type (workspace_id, name, schema, description, format_extension, display_name, edited_at)\n VALUES ('admins', $1, $2, $3, $4, $6, now())\n ON CONFLICT (workspace_id, name) DO UPDATE\n SET schema = EXCLUDED.schema, description = EXCLUDED.description,\n -- A fileset is a set of files, so it cannot also be one file.\n -- Create and update reject the pair; this writer bypasses both, so\n -- it declines the extension rather than persisting the forbidden\n -- combination onto a same-named local fileset.\n format_extension = CASE\n WHEN resource_type.is_fileset THEN NULL\n WHEN $5 THEN EXCLUDED.format_extension\n ELSE resource_type.format_extension END,\n display_name = CASE WHEN $7 THEN EXCLUDED.display_name ELSE resource_type.display_name END,\n edited_at = now()", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Varchar", - "Jsonb", - "Text", - "Varchar", - "Bool", - "Varchar", - "Bool" - ] - }, - "nullable": [] - }, - "hash": "212bf5b32de102a9c537907aaff04befc3a6596c258086805dd42f513ddb3ead" -} diff --git a/backend/.sqlx/query-24cfd719a45af5beb4eb264939290c01e1b684525bae4cb329515989cd341875.json b/backend/.sqlx/query-24cfd719a45af5beb4eb264939290c01e1b684525bae4cb329515989cd341875.json new file mode 100644 index 0000000000..eaa7d60dea --- /dev/null +++ b/backend/.sqlx/query-24cfd719a45af5beb4eb264939290c01e1b684525bae4cb329515989cd341875.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO runnable_job_stats\n (hour, workspace_id, runnable_path, worker_group, job_count, total_duration_ms,\n max_memory_peak, sum_memory_peak, memory_sample_count, total_cpu_ms)\n SELECT s.hour, s.workspace_id, s.runnable_path, s.worker_group, s.job_count,\n s.total_duration_ms, s.max_memory_peak, s.sum_memory_peak,\n s.memory_sample_count, s.total_cpu_ms\n FROM UNNEST($1::bigint[], $2::text[], $3::text[], $4::text[], $5::int[], $6::bigint[],\n $7::int[], $8::bigint[], $9::int[], $10::bigint[])\n AS s(hour, workspace_id, runnable_path, worker_group, job_count, total_duration_ms,\n max_memory_peak, sum_memory_peak, memory_sample_count, total_cpu_ms)\n JOIN workspace w ON w.id = s.workspace_id\n ORDER BY s.hour, s.workspace_id, s.runnable_path, s.worker_group\n ON CONFLICT (hour, workspace_id, runnable_path, worker_group) DO UPDATE SET\n job_count = runnable_job_stats.job_count + EXCLUDED.job_count,\n total_duration_ms = runnable_job_stats.total_duration_ms + EXCLUDED.total_duration_ms,\n max_memory_peak = GREATEST(runnable_job_stats.max_memory_peak, EXCLUDED.max_memory_peak),\n sum_memory_peak = runnable_job_stats.sum_memory_peak + EXCLUDED.sum_memory_peak,\n memory_sample_count = runnable_job_stats.memory_sample_count + EXCLUDED.memory_sample_count,\n total_cpu_ms = runnable_job_stats.total_cpu_ms + EXCLUDED.total_cpu_ms\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int8Array", + "TextArray", + "TextArray", + "TextArray", + "Int4Array", + "Int8Array", + "Int4Array", + "Int8Array", + "Int4Array", + "Int8Array" + ] + }, + "nullable": [] + }, + "hash": "24cfd719a45af5beb4eb264939290c01e1b684525bae4cb329515989cd341875" +} diff --git a/backend/.sqlx/query-333f7116d660756c36dfc0aa2b8d7ebd10a38cbe5724a554a12291c287b67f6d.json b/backend/.sqlx/query-333f7116d660756c36dfc0aa2b8d7ebd10a38cbe5724a554a12291c287b67f6d.json new file mode 100644 index 0000000000..23b46f5577 --- /dev/null +++ b/backend/.sqlx/query-333f7116d660756c36dfc0aa2b8d7ebd10a38cbe5724a554a12291c287b67f6d.json @@ -0,0 +1,324 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT\n v2_job_queue.workspace_id,\n v2_job_queue.id,\n v2_job.args as \"args: sqlx::types::Json>>\",\n v2_job.parent_job,\n v2_job.created_by,\n v2_job_queue.started_at,\n v2_job_queue.runnable_settings_handle,\n scheduled_for,\n runnable_path,\n kind as \"kind: JobKind\",\n runnable_id as \"runnable_id: ScriptHash\",\n canceled_reason,\n canceled_by,\n permissioned_as,\n permissioned_as_email,\n flow_status as \"flow_status: sqlx::types::Json>\",\n v2_job.tag,\n script_lang as \"script_lang: ScriptLang\",\n same_worker,\n pre_run_error,\n concurrent_limit,\n concurrency_time_window_s,\n flow_innermost_root_job,\n root_job,\n timeout,\n flow_step_id,\n cache_ttl,\n cache_ignore_s3_path,\n v2_job_queue.priority,\n preprocessed,\n script_entrypoint_override,\n trigger,\n trigger_kind as \"trigger_kind: TriggerKindLabel\",\n visible_to_owner,\n NULL as permissioned_as_end_user_email,\n job_perms.job_token_scopes\n FROM v2_job_queue INNER JOIN v2_job ON v2_job.id = v2_job_queue.id LEFT JOIN v2_job_status ON v2_job_status.id = v2_job_queue.id\n LEFT JOIN job_perms ON job_perms.job_id = v2_job_queue.id WHERE v2_job_queue.id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "workspace_id", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "id", + "type_info": "Uuid" + }, + { + "ordinal": 2, + "name": "args: sqlx::types::Json>>", + "type_info": "Jsonb" + }, + { + "ordinal": 3, + "name": "parent_job", + "type_info": "Uuid" + }, + { + "ordinal": 4, + "name": "created_by", + "type_info": "Varchar" + }, + { + "ordinal": 5, + "name": "started_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 6, + "name": "runnable_settings_handle", + "type_info": "Int8" + }, + { + "ordinal": 7, + "name": "scheduled_for", + "type_info": "Timestamptz" + }, + { + "ordinal": 8, + "name": "runnable_path", + "type_info": "Varchar" + }, + { + "ordinal": 9, + "name": "kind: JobKind", + "type_info": { + "Custom": { + "name": "job_kind", + "kind": { + "Enum": [ + "script", + "preview", + "flow", + "dependencies", + "flowpreview", + "script_hub", + "identity", + "flowdependencies", + "http", + "graphql", + "postgresql", + "noop", + "appdependencies", + "deploymentcallback", + "singlestepflow", + "flowscript", + "flownode", + "appscript", + "aiagent", + "unassigned_script", + "unassigned_flow", + "unassigned_singlestepflow" + ] + } + } + } + }, + { + "ordinal": 10, + "name": "runnable_id: ScriptHash", + "type_info": "Int8" + }, + { + "ordinal": 11, + "name": "canceled_reason", + "type_info": "Text" + }, + { + "ordinal": 12, + "name": "canceled_by", + "type_info": "Varchar" + }, + { + "ordinal": 13, + "name": "permissioned_as", + "type_info": "Varchar" + }, + { + "ordinal": 14, + "name": "permissioned_as_email", + "type_info": "Varchar" + }, + { + "ordinal": 15, + "name": "flow_status: sqlx::types::Json>", + "type_info": "Jsonb" + }, + { + "ordinal": 16, + "name": "tag", + "type_info": "Varchar" + }, + { + "ordinal": 17, + "name": "script_lang: ScriptLang", + "type_info": { + "Custom": { + "name": "script_lang", + "kind": { + "Enum": [ + "python3", + "deno", + "go", + "bash", + "postgresql", + "nativets", + "bun", + "mysql", + "bigquery", + "snowflake", + "graphql", + "powershell", + "mssql", + "php", + "bunnative", + "rust", + "ansible", + "csharp", + "oracledb", + "nu", + "java", + "duckdb", + "ruby", + "rlang", + "dbt" + ] + } + } + } + }, + { + "ordinal": 18, + "name": "same_worker", + "type_info": "Bool" + }, + { + "ordinal": 19, + "name": "pre_run_error", + "type_info": "Text" + }, + { + "ordinal": 20, + "name": "concurrent_limit", + "type_info": "Int4" + }, + { + "ordinal": 21, + "name": "concurrency_time_window_s", + "type_info": "Int4" + }, + { + "ordinal": 22, + "name": "flow_innermost_root_job", + "type_info": "Uuid" + }, + { + "ordinal": 23, + "name": "root_job", + "type_info": "Uuid" + }, + { + "ordinal": 24, + "name": "timeout", + "type_info": "Int4" + }, + { + "ordinal": 25, + "name": "flow_step_id", + "type_info": "Varchar" + }, + { + "ordinal": 26, + "name": "cache_ttl", + "type_info": "Int4" + }, + { + "ordinal": 27, + "name": "cache_ignore_s3_path", + "type_info": "Bool" + }, + { + "ordinal": 28, + "name": "priority", + "type_info": "Int2" + }, + { + "ordinal": 29, + "name": "preprocessed", + "type_info": "Bool" + }, + { + "ordinal": 30, + "name": "script_entrypoint_override", + "type_info": "Varchar" + }, + { + "ordinal": 31, + "name": "trigger", + "type_info": "Varchar" + }, + { + "ordinal": 32, + "name": "trigger_kind: TriggerKindLabel", + "type_info": { + "Custom": { + "name": "job_trigger_kind", + "kind": { + "Enum": [ + "webhook", + "http", + "websocket", + "kafka", + "email", + "nats", + "schedule", + "app", + "ui", + "postgres", + "sqs", + "gcp", + "mqtt", + "nextcloud", + "google", + "ci_test", + "github", + "azure", + "asset", + "freshness", + "amqp" + ] + } + } + } + }, + { + "ordinal": 33, + "name": "visible_to_owner", + "type_info": "Bool" + }, + { + "ordinal": 34, + "name": "permissioned_as_end_user_email", + "type_info": "Text" + }, + { + "ordinal": 35, + "name": "job_token_scopes", + "type_info": "TextArray" + } + ], + "parameters": { + "Left": [ + "Uuid" + ] + }, + "nullable": [ + false, + false, + true, + true, + false, + true, + true, + false, + true, + false, + true, + true, + true, + false, + false, + true, + false, + true, + false, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + false, + null, + true + ] + }, + "hash": "333f7116d660756c36dfc0aa2b8d7ebd10a38cbe5724a554a12291c287b67f6d" +} diff --git a/backend/.sqlx/query-7f98c013682aca6d269ef6ef754a7247b6e2dd4958d0968a3f2eb49cc320e877.json b/backend/.sqlx/query-344c08104d62d9bcd3f9117abb574e73a6fb364e4ab3e3916958c2e493673b1d.json similarity index 69% rename from backend/.sqlx/query-7f98c013682aca6d269ef6ef754a7247b6e2dd4958d0968a3f2eb49cc320e877.json rename to backend/.sqlx/query-344c08104d62d9bcd3f9117abb574e73a6fb364e4ab3e3916958c2e493673b1d.json index 90b559a9b6..37f090c2b4 100644 --- a/backend/.sqlx/query-7f98c013682aca6d269ef6ef754a7247b6e2dd4958d0968a3f2eb49cc320e877.json +++ b/backend/.sqlx/query-344c08104d62d9bcd3f9117abb574e73a6fb364e4ab3e3916958c2e493673b1d.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by, schema, summary, description, lock)\n VALUES ('test-workspace', 3333333333, 'f/test/error_handler', 'export function main() { return \"handled\"; }', 'deno', 'script', 'test-user', '{}', '', '', '')\n ", + "query": "\n INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by, schema, summary, description, lock)\n VALUES ('test-workspace', 3333333333, 'f/test/error_handler', 'export function main() { return \"handled\"; }', 'bun', 'script', 'test-user', '{}', '', '', E'{}\\n//bun.lock\\n')\n ", "describe": { "columns": [], "parameters": { @@ -8,5 +8,5 @@ }, "nullable": [] }, - "hash": "7f98c013682aca6d269ef6ef754a7247b6e2dd4958d0968a3f2eb49cc320e877" + "hash": "344c08104d62d9bcd3f9117abb574e73a6fb364e4ab3e3916958c2e493673b1d" } diff --git a/backend/.sqlx/query-35f960e9de880fdcaf81615933c828c4a877709c3c39a530d629cc91f9dd2d29.json b/backend/.sqlx/query-35f960e9de880fdcaf81615933c828c4a877709c3c39a530d629cc91f9dd2d29.json new file mode 100644 index 0000000000..db2af83b47 --- /dev/null +++ b/backend/.sqlx/query-35f960e9de880fdcaf81615933c828c4a877709c3c39a530d629cc91f9dd2d29.json @@ -0,0 +1,29 @@ +{ + "db_name": "PostgreSQL", + "query": "WITH RECURSIVE chain(origin, id, parent_job) AS (\n SELECT id, id, parent_job FROM v2_job WHERE id = ANY($1) AND workspace_id = $2\n UNION ALL\n SELECT c.origin, j.id, j.parent_job FROM v2_job j\n JOIN chain c ON j.id = c.parent_job AND j.workspace_id = $2\n )\n SELECT c.origin AS \"origin!\",\n CASE WHEN a.agent THEN regexp_replace(j.runnable_path, '\\.chat$', '')\n ELSE j.runnable_path END AS \"runnable_path!\"\n FROM chain c JOIN v2_job j ON j.id = c.id,\n LATERAL (SELECT j.kind = 'flowpreview'\n AND j.raw_flow->'modules'->1 IS NULL\n AND j.raw_flow->'modules'->0->>'id' = '__wm_agent_root' AS agent) a\n WHERE j.workspace_id = $2 AND j.runnable_path IS NOT NULL\n AND (a.agent OR j.kind IN ('script', 'script_hub', 'unassigned_script', 'flow',\n 'unassigned_flow', 'singlestepflow', 'unassigned_singlestepflow'))", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "origin!", + "type_info": "Uuid" + }, + { + "ordinal": 1, + "name": "runnable_path!", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "UuidArray", + "Text" + ] + }, + "nullable": [ + null, + null + ] + }, + "hash": "35f960e9de880fdcaf81615933c828c4a877709c3c39a530d629cc91f9dd2d29" +} diff --git a/backend/.sqlx/query-386e14cf7572027f2c4ef313cd7cc5dd6c7b0da4f76131313d3e6d97f36c5c34.json b/backend/.sqlx/query-386e14cf7572027f2c4ef313cd7cc5dd6c7b0da4f76131313d3e6d97f36c5c34.json deleted file mode 100644 index 2f794e6be0..0000000000 --- a/backend/.sqlx/query-386e14cf7572027f2c4ef313cd7cc5dd6c7b0da4f76131313d3e6d97f36c5c34.json +++ /dev/null @@ -1,28 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT EXISTS(SELECT 1 FROM resource_type WHERE workspace_id = 'admins' AND name = $1 AND schema IS NOT DISTINCT FROM $2 AND description IS NOT DISTINCT FROM $3 AND ($5 IS NOT TRUE OR format_extension IS NOT DISTINCT FROM $4) AND ($7 IS NOT TRUE OR display_name IS NOT DISTINCT FROM $6))", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "exists", - "type_info": "Bool" - } - ], - "parameters": { - "Left": [ - "Text", - "Jsonb", - "Text", - "Text", - "Bool", - "Text", - "Bool" - ] - }, - "nullable": [ - null - ] - }, - "hash": "386e14cf7572027f2c4ef313cd7cc5dd6c7b0da4f76131313d3e6d97f36c5c34" -} diff --git a/backend/.sqlx/query-2d43c1889b492df6f77380e739e41e20fe6bea0553cc39e30b7aa8ae59963f95.json b/backend/.sqlx/query-42250637493a623850ca76fe5a705fe4acef039f38a5e76da8d09f2f7e755779.json similarity index 58% rename from backend/.sqlx/query-2d43c1889b492df6f77380e739e41e20fe6bea0553cc39e30b7aa8ae59963f95.json rename to backend/.sqlx/query-42250637493a623850ca76fe5a705fe4acef039f38a5e76da8d09f2f7e755779.json index 532fc430ab..402a406820 100644 --- a/backend/.sqlx/query-2d43c1889b492df6f77380e739e41e20fe6bea0553cc39e30b7aa8ae59963f95.json +++ b/backend/.sqlx/query-42250637493a623850ca76fe5a705fe4acef039f38a5e76da8d09f2f7e755779.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by, schema, summary, description, lock)\n VALUES ('test-workspace', 1234567890, 'f/test/success_handler', $1, 'deno', 'script', 'test-user', '{}', 'Success handler script', 'Handles successful job completions', '')\n ", + "query": "\n INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by, schema, summary, description, lock)\n VALUES ('test-workspace', 1234567890, 'f/test/success_handler', $1, 'bun', 'script', 'test-user', '{}', 'Success handler script', 'Handles successful job completions', E'{}\\n//bun.lock\\n')\n ", "describe": { "columns": [], "parameters": { @@ -10,5 +10,5 @@ }, "nullable": [] }, - "hash": "2d43c1889b492df6f77380e739e41e20fe6bea0553cc39e30b7aa8ae59963f95" + "hash": "42250637493a623850ca76fe5a705fe4acef039f38a5e76da8d09f2f7e755779" } diff --git a/backend/.sqlx/query-4631200a37f7edec92f3796ab4d4585d9c2aba36fdf33bca9739049366bed107.json b/backend/.sqlx/query-4631200a37f7edec92f3796ab4d4585d9c2aba36fdf33bca9739049366bed107.json new file mode 100644 index 0000000000..a5bde4d661 --- /dev/null +++ b/backend/.sqlx/query-4631200a37f7edec92f3796ab4d4585d9c2aba36fdf33bca9739049366bed107.json @@ -0,0 +1,84 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT\n flow_version.id AS version,\n flow_version.value->>'early_return' as early_return,\n flow_version.value->>'preprocessor_module' IS NOT NULL as has_preprocessor,\n flow_version.value->>'failure_module' IS NOT NULL as has_failure_module,\n (flow_version.value->>'chat_input_enabled')::boolean as chat_input_enabled,\n flow.tag,\n flow.dedicated_worker,\n flow.on_behalf_of,\n flow.edited_by,\n flow.labels,\n flow.job_token_scopes\n FROM\n flow_version\n INNER JOIN flow\n ON flow.path = flow_version.path AND\n flow.workspace_id = flow_version.workspace_id\n WHERE\n flow_version.workspace_id = $1 AND\n flow_version.path = $2 AND\n flow_version.id = $3\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "version", + "type_info": "Int8" + }, + { + "ordinal": 1, + "name": "early_return", + "type_info": "Text" + }, + { + "ordinal": 2, + "name": "has_preprocessor", + "type_info": "Bool" + }, + { + "ordinal": 3, + "name": "has_failure_module", + "type_info": "Bool" + }, + { + "ordinal": 4, + "name": "chat_input_enabled", + "type_info": "Bool" + }, + { + "ordinal": 5, + "name": "tag", + "type_info": "Varchar" + }, + { + "ordinal": 6, + "name": "dedicated_worker", + "type_info": "Bool" + }, + { + "ordinal": 7, + "name": "on_behalf_of", + "type_info": "Varchar" + }, + { + "ordinal": 8, + "name": "edited_by", + "type_info": "Varchar" + }, + { + "ordinal": 9, + "name": "labels", + "type_info": "TextArray" + }, + { + "ordinal": 10, + "name": "job_token_scopes", + "type_info": "TextArray" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Int8" + ] + }, + "nullable": [ + false, + null, + null, + null, + null, + true, + true, + true, + false, + true, + true + ] + }, + "hash": "4631200a37f7edec92f3796ab4d4585d9c2aba36fdf33bca9739049366bed107" +} diff --git a/backend/.sqlx/query-4cc81f5e04caefa4f170807c0ce96c69f22d9ea9dc3586099553a8f0e76aad82.json b/backend/.sqlx/query-4cc81f5e04caefa4f170807c0ce96c69f22d9ea9dc3586099553a8f0e76aad82.json new file mode 100644 index 0000000000..7051c33008 --- /dev/null +++ b/backend/.sqlx/query-4cc81f5e04caefa4f170807c0ce96c69f22d9ea9dc3586099553a8f0e76aad82.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO job_perms (job_id, email, username, is_admin, is_operator, folders, groups, workspace_id, job_token_scopes)\n SELECT unnest($1::uuid[]), $2, $3, $4, $5, $6, $7, $8, $9", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "UuidArray", + "Varchar", + "Varchar", + "Bool", + "Bool", + "JsonbArray", + "TextArray", + "Varchar", + "TextArray" + ] + }, + "nullable": [] + }, + "hash": "4cc81f5e04caefa4f170807c0ce96c69f22d9ea9dc3586099553a8f0e76aad82" +} diff --git a/backend/.sqlx/query-580271c5fe178f5e31f3e211cd7fea97aec4d6d3ec65d3e7402811c7dddbbf64.json b/backend/.sqlx/query-580271c5fe178f5e31f3e211cd7fea97aec4d6d3ec65d3e7402811c7dddbbf64.json new file mode 100644 index 0000000000..e1f4609346 --- /dev/null +++ b/backend/.sqlx/query-580271c5fe178f5e31f3e211cd7fea97aec4d6d3ec65d3e7402811c7dddbbf64.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO flow\n (workspace_id, path, summary, description, archived, extra_perms, dependency_job, tag, ws_error_handler_muted, dedicated_worker, timeout, visible_to_runner_only, on_behalf_of, on_behalf_of_email, concurrency_key, versions, value, schema, edited_by, edited_at, labels, job_token_scopes)\n SELECT workspace_id, $1, summary, description, archived, extra_perms, dependency_job, tag, ws_error_handler_muted, dedicated_worker, timeout, visible_to_runner_only, on_behalf_of, on_behalf_of_email, concurrency_key, versions, value, schema, edited_by, edited_at, labels, job_token_scopes\n FROM flow\n WHERE path = $2 AND workspace_id = $3", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "580271c5fe178f5e31f3e211cd7fea97aec4d6d3ec65d3e7402811c7dddbbf64" +} diff --git a/backend/.sqlx/query-58ea7e2c6b07ffc7ab7387b2ebb542cba3a5532c64a9594d2e0ccdf21dee90d0.json b/backend/.sqlx/query-58ea7e2c6b07ffc7ab7387b2ebb542cba3a5532c64a9594d2e0ccdf21dee90d0.json new file mode 100644 index 0000000000..e03d995f15 --- /dev/null +++ b/backend/.sqlx/query-58ea7e2c6b07ffc7ab7387b2ebb542cba3a5532c64a9594d2e0ccdf21dee90d0.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT canceled_by IS NOT NULL FROM v2_job_queue WHERE id = $1 AND workspace_id = $2 AND running", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "?column?", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "58ea7e2c6b07ffc7ab7387b2ebb542cba3a5532c64a9594d2e0ccdf21dee90d0" +} diff --git a/backend/.sqlx/query-5a975e486a0cdb0fae4dadd344ad3abc2f67827597ef74fef473e309b1951544.json b/backend/.sqlx/query-5a975e486a0cdb0fae4dadd344ad3abc2f67827597ef74fef473e309b1951544.json new file mode 100644 index 0000000000..56f5ddbc84 --- /dev/null +++ b/backend/.sqlx/query-5a975e486a0cdb0fae4dadd344ad3abc2f67827597ef74fef473e309b1951544.json @@ -0,0 +1,253 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT\n j.id, j.workspace_id, j.runnable_id AS \"runnable_id: ScriptHash\", q.scheduled_for, q.started_at, j.parent_job, j.flow_innermost_root_job, j.runnable_path, j.kind as \"kind!: JobKind\", j.permissioned_as,\n j.created_by, j.script_lang AS \"script_lang: ScriptLang\", j.permissioned_as_email, j.flow_step_id, j.trigger_kind AS \"trigger_kind: TriggerKindLabel\", j.trigger, j.priority, j.concurrent_limit, j.tag, j.cache_ttl, q.cache_ignore_s3_path, q.runnable_settings_handle,\n COALESCE(j.args->'build_binary_only' = 'true'::jsonb, false) AS \"build_binary_only!\",\n p.job_token_scopes AS \"job_token_scopes?\"\n FROM v2_job j LEFT JOIN v2_job_queue q ON j.id = q.id\n LEFT JOIN job_perms p ON p.job_id = j.id\n WHERE j.id = $1 AND j.workspace_id = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Uuid" + }, + { + "ordinal": 1, + "name": "workspace_id", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "runnable_id: ScriptHash", + "type_info": "Int8" + }, + { + "ordinal": 3, + "name": "scheduled_for", + "type_info": "Timestamptz" + }, + { + "ordinal": 4, + "name": "started_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 5, + "name": "parent_job", + "type_info": "Uuid" + }, + { + "ordinal": 6, + "name": "flow_innermost_root_job", + "type_info": "Uuid" + }, + { + "ordinal": 7, + "name": "runnable_path", + "type_info": "Varchar" + }, + { + "ordinal": 8, + "name": "kind!: JobKind", + "type_info": { + "Custom": { + "name": "job_kind", + "kind": { + "Enum": [ + "script", + "preview", + "flow", + "dependencies", + "flowpreview", + "script_hub", + "identity", + "flowdependencies", + "http", + "graphql", + "postgresql", + "noop", + "appdependencies", + "deploymentcallback", + "singlestepflow", + "flowscript", + "flownode", + "appscript", + "aiagent", + "unassigned_script", + "unassigned_flow", + "unassigned_singlestepflow" + ] + } + } + } + }, + { + "ordinal": 9, + "name": "permissioned_as", + "type_info": "Varchar" + }, + { + "ordinal": 10, + "name": "created_by", + "type_info": "Varchar" + }, + { + "ordinal": 11, + "name": "script_lang: ScriptLang", + "type_info": { + "Custom": { + "name": "script_lang", + "kind": { + "Enum": [ + "python3", + "deno", + "go", + "bash", + "postgresql", + "nativets", + "bun", + "mysql", + "bigquery", + "snowflake", + "graphql", + "powershell", + "mssql", + "php", + "bunnative", + "rust", + "ansible", + "csharp", + "oracledb", + "nu", + "java", + "duckdb", + "ruby", + "rlang", + "dbt" + ] + } + } + } + }, + { + "ordinal": 12, + "name": "permissioned_as_email", + "type_info": "Varchar" + }, + { + "ordinal": 13, + "name": "flow_step_id", + "type_info": "Varchar" + }, + { + "ordinal": 14, + "name": "trigger_kind: TriggerKindLabel", + "type_info": { + "Custom": { + "name": "job_trigger_kind", + "kind": { + "Enum": [ + "webhook", + "http", + "websocket", + "kafka", + "email", + "nats", + "schedule", + "app", + "ui", + "postgres", + "sqs", + "gcp", + "mqtt", + "nextcloud", + "google", + "ci_test", + "github", + "azure", + "asset", + "freshness", + "amqp" + ] + } + } + } + }, + { + "ordinal": 15, + "name": "trigger", + "type_info": "Varchar" + }, + { + "ordinal": 16, + "name": "priority", + "type_info": "Int2" + }, + { + "ordinal": 17, + "name": "concurrent_limit", + "type_info": "Int4" + }, + { + "ordinal": 18, + "name": "tag", + "type_info": "Varchar" + }, + { + "ordinal": 19, + "name": "cache_ttl", + "type_info": "Int4" + }, + { + "ordinal": 20, + "name": "cache_ignore_s3_path", + "type_info": "Bool" + }, + { + "ordinal": 21, + "name": "runnable_settings_handle", + "type_info": "Int8" + }, + { + "ordinal": 22, + "name": "build_binary_only!", + "type_info": "Bool" + }, + { + "ordinal": 23, + "name": "job_token_scopes?", + "type_info": "TextArray" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Text" + ] + }, + "nullable": [ + false, + false, + true, + false, + true, + true, + true, + true, + false, + false, + false, + true, + false, + true, + true, + true, + true, + true, + false, + true, + true, + true, + null, + true + ] + }, + "hash": "5a975e486a0cdb0fae4dadd344ad3abc2f67827597ef74fef473e309b1951544" +} diff --git a/backend/.sqlx/query-60d53c73e67dd7b29cce910fd76b6c87f2994fa2d5f33e0f3ee9effe28614abe.json b/backend/.sqlx/query-60d53c73e67dd7b29cce910fd76b6c87f2994fa2d5f33e0f3ee9effe28614abe.json new file mode 100644 index 0000000000..f04f994e6a --- /dev/null +++ b/backend/.sqlx/query-60d53c73e67dd7b29cce910fd76b6c87f2994fa2d5f33e0f3ee9effe28614abe.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT job_token_scopes FROM script WHERE hash = $1 AND workspace_id = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "job_token_scopes", + "type_info": "TextArray" + } + ], + "parameters": { + "Left": [ + "Int8", + "Text" + ] + }, + "nullable": [ + true + ] + }, + "hash": "60d53c73e67dd7b29cce910fd76b6c87f2994fa2d5f33e0f3ee9effe28614abe" +} diff --git a/backend/.sqlx/query-6445041033c95fcf52c79c33f34cf6c41fb443f8bd91f12bce0e53f6e91bfb16.json b/backend/.sqlx/query-6445041033c95fcf52c79c33f34cf6c41fb443f8bd91f12bce0e53f6e91bfb16.json new file mode 100644 index 0000000000..abcdf7baab --- /dev/null +++ b/backend/.sqlx/query-6445041033c95fcf52c79c33f34cf6c41fb443f8bd91f12bce0e53f6e91bfb16.json @@ -0,0 +1,331 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE v2_job_status SET\n flow_status = JSONB_SET(\n JSONB_SET(v2_job_status.flow_status, $2::TEXT[], $3),\n $4::TEXT[], $5\n ) - $6::TEXT[],\n flow_leaf_jobs = CASE\n WHEN $7::TEXT IS NULL\n OR COALESCE(v2_job.flow_innermost_root_job, v2_job_status.id) <> v2_job_status.id\n THEN v2_job_status.flow_leaf_jobs\n ELSE JSONB_SET(COALESCE(v2_job_status.flow_leaf_jobs, '{}'::JSONB), ARRAY[$7::TEXT], $8) END\n FROM v2_job_queue INNER JOIN v2_job ON v2_job.id = v2_job_queue.id\n LEFT JOIN job_perms ON job_perms.job_id = v2_job_queue.id\n WHERE v2_job_status.id = $1 AND v2_job_queue.id = $1\n RETURNING\n v2_job_queue.workspace_id,\n v2_job_queue.id,\n v2_job.args as \"args: sqlx::types::Json>>\",\n v2_job.parent_job,\n v2_job.created_by,\n v2_job_queue.started_at,\n v2_job_queue.runnable_settings_handle,\n v2_job_queue.scheduled_for,\n v2_job.runnable_path,\n v2_job.kind as \"kind: JobKind\",\n v2_job.runnable_id as \"runnable_id: ScriptHash\",\n v2_job_queue.canceled_reason,\n v2_job_queue.canceled_by,\n v2_job.permissioned_as,\n v2_job.permissioned_as_email,\n v2_job_status.flow_status as \"flow_status: sqlx::types::Json>\",\n v2_job.tag,\n v2_job.script_lang as \"script_lang: ScriptLang\",\n v2_job.same_worker,\n v2_job.pre_run_error,\n v2_job.concurrent_limit,\n v2_job.concurrency_time_window_s,\n v2_job.flow_innermost_root_job,\n v2_job.root_job,\n v2_job.timeout,\n v2_job.flow_step_id,\n v2_job.cache_ttl,\n v2_job_queue.cache_ignore_s3_path,\n v2_job_queue.priority,\n v2_job.preprocessed,\n v2_job.script_entrypoint_override,\n v2_job.trigger,\n v2_job.trigger_kind as \"trigger_kind: TriggerKindLabel\",\n v2_job.visible_to_owner,\n NULL as permissioned_as_end_user_email,\n job_perms.job_token_scopes", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "workspace_id", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "id", + "type_info": "Uuid" + }, + { + "ordinal": 2, + "name": "args: sqlx::types::Json>>", + "type_info": "Jsonb" + }, + { + "ordinal": 3, + "name": "parent_job", + "type_info": "Uuid" + }, + { + "ordinal": 4, + "name": "created_by", + "type_info": "Varchar" + }, + { + "ordinal": 5, + "name": "started_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 6, + "name": "runnable_settings_handle", + "type_info": "Int8" + }, + { + "ordinal": 7, + "name": "scheduled_for", + "type_info": "Timestamptz" + }, + { + "ordinal": 8, + "name": "runnable_path", + "type_info": "Varchar" + }, + { + "ordinal": 9, + "name": "kind: JobKind", + "type_info": { + "Custom": { + "name": "job_kind", + "kind": { + "Enum": [ + "script", + "preview", + "flow", + "dependencies", + "flowpreview", + "script_hub", + "identity", + "flowdependencies", + "http", + "graphql", + "postgresql", + "noop", + "appdependencies", + "deploymentcallback", + "singlestepflow", + "flowscript", + "flownode", + "appscript", + "aiagent", + "unassigned_script", + "unassigned_flow", + "unassigned_singlestepflow" + ] + } + } + } + }, + { + "ordinal": 10, + "name": "runnable_id: ScriptHash", + "type_info": "Int8" + }, + { + "ordinal": 11, + "name": "canceled_reason", + "type_info": "Text" + }, + { + "ordinal": 12, + "name": "canceled_by", + "type_info": "Varchar" + }, + { + "ordinal": 13, + "name": "permissioned_as", + "type_info": "Varchar" + }, + { + "ordinal": 14, + "name": "permissioned_as_email", + "type_info": "Varchar" + }, + { + "ordinal": 15, + "name": "flow_status: sqlx::types::Json>", + "type_info": "Jsonb" + }, + { + "ordinal": 16, + "name": "tag", + "type_info": "Varchar" + }, + { + "ordinal": 17, + "name": "script_lang: ScriptLang", + "type_info": { + "Custom": { + "name": "script_lang", + "kind": { + "Enum": [ + "python3", + "deno", + "go", + "bash", + "postgresql", + "nativets", + "bun", + "mysql", + "bigquery", + "snowflake", + "graphql", + "powershell", + "mssql", + "php", + "bunnative", + "rust", + "ansible", + "csharp", + "oracledb", + "nu", + "java", + "duckdb", + "ruby", + "rlang", + "dbt" + ] + } + } + } + }, + { + "ordinal": 18, + "name": "same_worker", + "type_info": "Bool" + }, + { + "ordinal": 19, + "name": "pre_run_error", + "type_info": "Text" + }, + { + "ordinal": 20, + "name": "concurrent_limit", + "type_info": "Int4" + }, + { + "ordinal": 21, + "name": "concurrency_time_window_s", + "type_info": "Int4" + }, + { + "ordinal": 22, + "name": "flow_innermost_root_job", + "type_info": "Uuid" + }, + { + "ordinal": 23, + "name": "root_job", + "type_info": "Uuid" + }, + { + "ordinal": 24, + "name": "timeout", + "type_info": "Int4" + }, + { + "ordinal": 25, + "name": "flow_step_id", + "type_info": "Varchar" + }, + { + "ordinal": 26, + "name": "cache_ttl", + "type_info": "Int4" + }, + { + "ordinal": 27, + "name": "cache_ignore_s3_path", + "type_info": "Bool" + }, + { + "ordinal": 28, + "name": "priority", + "type_info": "Int2" + }, + { + "ordinal": 29, + "name": "preprocessed", + "type_info": "Bool" + }, + { + "ordinal": 30, + "name": "script_entrypoint_override", + "type_info": "Varchar" + }, + { + "ordinal": 31, + "name": "trigger", + "type_info": "Varchar" + }, + { + "ordinal": 32, + "name": "trigger_kind: TriggerKindLabel", + "type_info": { + "Custom": { + "name": "job_trigger_kind", + "kind": { + "Enum": [ + "webhook", + "http", + "websocket", + "kafka", + "email", + "nats", + "schedule", + "app", + "ui", + "postgres", + "sqs", + "gcp", + "mqtt", + "nextcloud", + "google", + "ci_test", + "github", + "azure", + "asset", + "freshness", + "amqp" + ] + } + } + } + }, + { + "ordinal": 33, + "name": "visible_to_owner", + "type_info": "Bool" + }, + { + "ordinal": 34, + "name": "permissioned_as_end_user_email", + "type_info": "Text" + }, + { + "ordinal": 35, + "name": "job_token_scopes", + "type_info": "TextArray" + } + ], + "parameters": { + "Left": [ + "Uuid", + "TextArray", + "Jsonb", + "TextArray", + "Jsonb", + "TextArray", + "Text", + "Jsonb" + ] + }, + "nullable": [ + false, + false, + true, + true, + false, + true, + true, + false, + true, + false, + true, + true, + true, + false, + false, + true, + false, + true, + false, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + false, + null, + true + ] + }, + "hash": "6445041033c95fcf52c79c33f34cf6c41fb443f8bd91f12bce0e53f6e91bfb16" +} diff --git a/backend/.sqlx/query-67db536d06c57f195641f178ce651cbd7b8896984115d55318bb6f4100d72d9a.json b/backend/.sqlx/query-67db536d06c57f195641f178ce651cbd7b8896984115d55318bb6f4100d72d9a.json new file mode 100644 index 0000000000..cfd88ed5cb --- /dev/null +++ b/backend/.sqlx/query-67db536d06c57f195641f178ce651cbd7b8896984115d55318bb6f4100d72d9a.json @@ -0,0 +1,24 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT $2 IN (parent_job, root_job, flow_innermost_root_job) FROM v2_job\n WHERE id = $1 AND workspace_id = $3", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "?column?", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Uuid", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "67db536d06c57f195641f178ce651cbd7b8896984115d55318bb6f4100d72d9a" +} diff --git a/backend/.sqlx/query-fe17aff543f28cf4d6d09ea7c4c7d18ff6a95a8402468131f5a5f33df91c59e4.json b/backend/.sqlx/query-69e1c04c246c42a3a1d7202b6fb349d0ac779431993e225945ddfb91489cb927.json similarity index 59% rename from backend/.sqlx/query-fe17aff543f28cf4d6d09ea7c4c7d18ff6a95a8402468131f5a5f33df91c59e4.json rename to backend/.sqlx/query-69e1c04c246c42a3a1d7202b6fb349d0ac779431993e225945ddfb91489cb927.json index c747e99795..751b727d0f 100644 --- a/backend/.sqlx/query-fe17aff543f28cf4d6d09ea7c4c7d18ff6a95a8402468131f5a5f33df91c59e4.json +++ b/backend/.sqlx/query-69e1c04c246c42a3a1d7202b6fb349d0ac779431993e225945ddfb91489cb927.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by, schema, summary, description, lock)\n VALUES ('test-workspace', 1111111111, 'f/test/error_handler', $1, 'deno', 'script', 'test-user', '{}', 'Error handler script', 'Handles failed job completions', '')\n ", + "query": "\n INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by, schema, summary, description, lock)\n VALUES ('test-workspace', 1111111111, 'f/test/error_handler', $1, 'bun', 'script', 'test-user', '{}', 'Error handler script', 'Handles failed job completions', E'{}\\n//bun.lock\\n')\n ", "describe": { "columns": [], "parameters": { @@ -10,5 +10,5 @@ }, "nullable": [] }, - "hash": "fe17aff543f28cf4d6d09ea7c4c7d18ff6a95a8402468131f5a5f33df91c59e4" + "hash": "69e1c04c246c42a3a1d7202b6fb349d0ac779431993e225945ddfb91489cb927" } diff --git a/backend/.sqlx/query-706f2227e34971d22a577b33f6aaa7f5755da9997419c4a9d07b30d4da318bea.json b/backend/.sqlx/query-706f2227e34971d22a577b33f6aaa7f5755da9997419c4a9d07b30d4da318bea.json new file mode 100644 index 0000000000..7c305d1e7a --- /dev/null +++ b/backend/.sqlx/query-706f2227e34971d22a577b33f6aaa7f5755da9997419c4a9d07b30d4da318bea.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO script (\n workspace_id, hash, path, parent_hashes, summary, description, content,\n created_by, created_at, archived, schema, deleted, is_template,\n extra_perms, lock, lock_error_logs, language, kind, tag,\n envs, concurrent_limit, concurrency_time_window_s, cache_ttl,\n dedicated_worker, ws_error_handler_muted, priority, timeout,\n delete_after_use, delete_after_secs, restart_unless_cancelled, concurrency_key,\n visible_to_runner_only, auto_kind, codebase, has_preprocessor,\n on_behalf_of, on_behalf_of_email, assets, modules, job_token_scopes\n )\n SELECT\n $1, hash, path, parent_hashes, summary, description, content,\n created_by, created_at, archived, schema, deleted, is_template,\n extra_perms, lock, lock_error_logs, language, kind, tag,\n envs, concurrent_limit, concurrency_time_window_s, cache_ttl,\n dedicated_worker, ws_error_handler_muted, priority, timeout,\n delete_after_use, delete_after_secs, restart_unless_cancelled, concurrency_key,\n visible_to_runner_only, auto_kind, codebase, has_preprocessor,\n -- Same three forms and the same prefix-first rule as permissioned_as_exists,\n -- superadmin fallback included: one acting outside their workspaces has no usr\n -- row but still authenticates.\n CASE WHEN on_behalf_of LIKE 'u/%' THEN\n (SELECT on_behalf_of WHERE EXISTS (\n SELECT 1 FROM usr u WHERE u.workspace_id = $1::varchar\n AND u.username = substring(on_behalf_of from 3)\n UNION ALL\n SELECT 1 FROM password p WHERE p.super_admin\n AND (p.username = substring(on_behalf_of from 3)\n OR p.email = substring(on_behalf_of from 3))))\n WHEN on_behalf_of LIKE 'g/%' THEN\n (SELECT on_behalf_of WHERE EXISTS (\n SELECT 1 FROM group_ g WHERE g.workspace_id = $1::varchar\n AND g.name = substring(on_behalf_of from 3)))\n ELSE\n (SELECT on_behalf_of WHERE EXISTS (\n SELECT 1 FROM usr u WHERE u.workspace_id = $1::varchar\n AND u.username = on_behalf_of\n UNION ALL\n SELECT 1 FROM password p WHERE p.email = on_behalf_of\n AND p.super_admin))\n END, on_behalf_of_email, assets, modules, job_token_scopes\n FROM script\n WHERE workspace_id = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Text" + ] + }, + "nullable": [] + }, + "hash": "706f2227e34971d22a577b33f6aaa7f5755da9997419c4a9d07b30d4da318bea" +} diff --git a/backend/.sqlx/query-71c9e0c0f1d9eba0f24c8e410ca2f2a64dc559e3228d3b763f68b1ad3491ae6e.json b/backend/.sqlx/query-71c9e0c0f1d9eba0f24c8e410ca2f2a64dc559e3228d3b763f68b1ad3491ae6e.json new file mode 100644 index 0000000000..ba1cc13fac --- /dev/null +++ b/backend/.sqlx/query-71c9e0c0f1d9eba0f24c8e410ca2f2a64dc559e3228d3b763f68b1ad3491ae6e.json @@ -0,0 +1,45 @@ +{ + "db_name": "PostgreSQL", + "query": "WITH deleted AS (\n DELETE FROM v2_job_queue WHERE id = $1\n RETURNING id, workspace_id, started_at, worker, canceled_by, canceled_reason,\n (extras->>'wac_first_started_at')::timestamptz AS first_started_at,\n COALESCE($9::bigint, (EXTRACT('epoch' FROM (now())) - EXTRACT('epoch' FROM (COALESCE(started_at, now()))))*1000)::bigint AS run_ms\n ), completed AS (\n INSERT INTO v2_job_completed AS cj\n ( workspace_id\n , id\n , started_at\n , duration_ms\n , extras\n , result\n , result_columns\n , canceled_by\n , canceled_reason\n , flow_status\n , workflow_as_code_status\n , memory_peak\n , status\n , worker\n )\n SELECT d.workspace_id, d.id, COALESCE(d.first_started_at, d.started_at),\n CASE WHEN d.first_started_at IS NULL THEN d.run_ms\n ELSE EXTRACT('epoch' FROM (now() - d.first_started_at))*1000 END,\n CASE WHEN d.first_started_at IS NOT NULL\n THEN jsonb_build_object('wac_last_segment_ms', d.run_ms) END,\n $3::text::jsonb, $10,\n CASE WHEN $4::BOOL THEN $5 ELSE d.canceled_by END,\n CASE WHEN $4::BOOL THEN $6 WHEN d.canceled_by IS NOT NULL THEN d.canceled_reason END,\n s.flow_status, s.workflow_as_code_status, $8,\n CASE WHEN $4::BOOL OR d.canceled_by IS NOT NULL THEN 'canceled'::job_status\n WHEN $7::BOOL THEN 'skipped'::job_status\n WHEN $2::BOOL THEN 'success'::job_status\n ELSE 'failure'::job_status END,\n d.worker\n FROM deleted d LEFT JOIN v2_job_status s ON s.id = d.id\n ON CONFLICT (id) DO UPDATE SET status = EXCLUDED.status, result = $3::text::jsonb,\n canceled_by = CASE WHEN NOT $4::BOOL AND EXCLUDED.canceled_by IS NOT NULL\n THEN EXCLUDED.canceled_by ELSE cj.canceled_by END,\n canceled_reason = CASE WHEN NOT $4::BOOL AND EXCLUDED.canceled_by IS NOT NULL\n THEN EXCLUDED.canceled_reason ELSE cj.canceled_reason END\n RETURNING cj.started_at, cj.duration_ms,\n COALESCE((cj.extras->>'wac_last_segment_ms')::bigint, cj.duration_ms) AS last_run_ms\n ), parent_ping AS (\n UPDATE v2_job_runtime r SET ping = now()\n FROM v2_job_queue q\n WHERE r.id = $11 AND q.id = r.id AND q.workspace_id = $12\n AND EXISTS (SELECT 1 FROM completed)\n )\n SELECT started_at AS \"started_at?\", duration_ms AS \"duration_ms!\",\n last_run_ms AS \"last_run_ms!\" FROM completed", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "started_at?", + "type_info": "Timestamptz" + }, + { + "ordinal": 1, + "name": "duration_ms!", + "type_info": "Int8" + }, + { + "ordinal": 2, + "name": "last_run_ms!", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Bool", + "Text", + "Bool", + "Varchar", + "Text", + "Bool", + "Int4", + "Int8", + "TextArray", + "Uuid", + "Text" + ] + }, + "nullable": [ + true, + false, + null + ] + }, + "hash": "71c9e0c0f1d9eba0f24c8e410ca2f2a64dc559e3228d3b763f68b1ad3491ae6e" +} diff --git a/backend/.sqlx/query-71de038f332593e2177f33284854b6c0bfe9bb4999f2c4e9082a4c5f3b341496.json b/backend/.sqlx/query-71de038f332593e2177f33284854b6c0bfe9bb4999f2c4e9082a4c5f3b341496.json deleted file mode 100644 index b647ec29fa..0000000000 --- a/backend/.sqlx/query-71de038f332593e2177f33284854b6c0bfe9bb4999f2c4e9082a4c5f3b341496.json +++ /dev/null @@ -1,38 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n SELECT\n id,\n runnable_path,\n permissioned_as_email,\n parent_job\n FROM v2_job\n WHERE workspace_id = 'test-workspace'\n AND permissioned_as_email = 'success_handler@windmill.dev'\n ORDER BY created_at DESC\n LIMIT 1\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "id", - "type_info": "Uuid" - }, - { - "ordinal": 1, - "name": "runnable_path", - "type_info": "Varchar" - }, - { - "ordinal": 2, - "name": "permissioned_as_email", - "type_info": "Varchar" - }, - { - "ordinal": 3, - "name": "parent_job", - "type_info": "Uuid" - } - ], - "parameters": { - "Left": [] - }, - "nullable": [ - false, - true, - false, - true - ] - }, - "hash": "71de038f332593e2177f33284854b6c0bfe9bb4999f2c4e9082a4c5f3b341496" -} diff --git a/backend/.sqlx/query-76440a1c5b40a897078817cc960247270c9044b363b71496ba36e1357c63a2a4.json b/backend/.sqlx/query-76440a1c5b40a897078817cc960247270c9044b363b71496ba36e1357c63a2a4.json new file mode 100644 index 0000000000..8eecbfa7bc --- /dev/null +++ b/backend/.sqlx/query-76440a1c5b40a897078817cc960247270c9044b363b71496ba36e1357c63a2a4.json @@ -0,0 +1,155 @@ +{ + "db_name": "PostgreSQL", + "query": "WITH RECURSIVE lineage AS (\n SELECT id, parent_job, kind, runnable_path, runnable_id, trigger_kind, trigger,\n tag, permissioned_as, args, 0 AS depth\n FROM v2_job WHERE id = $1 AND workspace_id = $2\n UNION ALL\n SELECT p.id, p.parent_job, p.kind, p.runnable_path, p.runnable_id, p.trigger_kind,\n p.trigger, p.tag, p.permissioned_as, p.args, l.depth + 1\n FROM v2_job p JOIN lineage l ON p.id = l.parent_job\n WHERE p.workspace_id = $2 AND l.depth < 100\n )\n SELECT id AS \"id!\", parent_job, kind AS \"kind!: JobKind\", runnable_path, runnable_id,\n trigger_kind::text AS trigger_kind, trigger, tag AS \"tag!\",\n permissioned_as AS \"permissioned_as!\",\n -- A restart takes its flow version from the request, so a trusted flow path\n -- can carry another flow's code: the version must belong to that path.\n CASE kind\n WHEN 'flow' THEN EXISTS (SELECT 1 FROM flow_version fv\n WHERE fv.id = runnable_id AND fv.path = runnable_path AND fv.workspace_id = $2)\n WHEN 'script' THEN EXISTS (SELECT 1 FROM script s\n WHERE s.hash = runnable_id AND s.path = runnable_path AND s.workspace_id = $2\n AND NOT s.deleted)\n ELSE true\n END AS \"origin_verified!\",\n -- What `run/f` and `run/p` resolve the path to now; for scripts the predicate of\n -- `get_latest_deployed_script_hash`.\n COALESCE(CASE kind\n WHEN 'flow' THEN runnable_id = (SELECT f.versions[array_upper(f.versions, 1)]\n FROM flow f WHERE f.path = runnable_path AND f.workspace_id = $2)\n WHEN 'script' THEN runnable_id = (SELECT s.hash FROM script s\n WHERE s.path = runnable_path AND s.workspace_id = $2 AND NOT s.deleted\n AND s.lock IS NOT NULL AND s.lock_error_logs IS NULL\n ORDER BY s.created_at DESC LIMIT 1)\n END, false) AS \"current_version!\",\n EXISTS (SELECT 1 FROM v2_job_status st WHERE st.id = lineage.id\n AND jsonb_typeof(st.flow_status->'restarted_from') = 'object')\n OR EXISTS (SELECT 1 FROM v2_job_completed c WHERE c.id = lineage.id\n AND jsonb_typeof(c.flow_status->'restarted_from') = 'object')\n AS \"restarted!\",\n COALESCE(\n (SELECT st.flow_status->'restarted_from'->>'flow_job_id' FROM v2_job_status st\n WHERE st.id = lineage.id),\n (SELECT c.flow_status->'restarted_from'->>'flow_job_id' FROM v2_job_completed c\n WHERE c.id = lineage.id)\n ) AS restarted_from,\n -- Only deployed-app runs are stamped with their app; an app editor preview\n -- runs app code at an app path it does not have to own.\n COALESCE(trigger_kind = 'app' AND starts_with(runnable_path, trigger || '/'), false)\n AS \"app_stamped!\",\n -- A preview's modules come from its args, which its parent may have taken from\n -- the caller.\n COALESCE(jsonb_typeof(args->'_MODULES') = 'object', false) AS \"args_modules!\",\n CASE WHEN depth = 0 THEN (SELECT wp.worker_group FROM v2_job_queue q\n JOIN worker_ping wp ON wp.worker = q.worker WHERE q.id = lineage.id)\n END AS worker_group,\n CASE WHEN depth = 0 THEN CASE\n WHEN starts_with(permissioned_as, 'g/') THEN 'group'\n ELSE (SELECT CASE WHEN u.is_service_account THEN 'service_account' ELSE 'user' END\n FROM usr u WHERE starts_with(permissioned_as, 'u/')\n AND u.username = substr(permissioned_as, 3) AND u.workspace_id = $2)\n END END AS run_as_type,\n -- The canonical item `item_digest` hashes, null members included.\n CASE WHEN depth = 0 OR depth = max(depth) OVER () THEN CASE kind\n WHEN 'script' THEN (SELECT jsonb_build_object('content', s.content,\n 'lock', s.lock, 'codebase', s.codebase,\n 'modules', (SELECT jsonb_object_agg(m.key, m.value - 'language')\n FROM jsonb_each(s.modules) m))\n FROM script s WHERE s.hash = runnable_id AND s.workspace_id = $2\n AND NOT s.deleted)\n WHEN 'flow' THEN (SELECT fv.value FROM flow_version fv\n WHERE fv.id = runnable_id AND fv.workspace_id = $2)\n WHEN 'flowscript' THEN (SELECT jsonb_build_object('content', n.code,\n 'lock', n.lock)\n FROM flow_node n WHERE n.id = runnable_id AND n.workspace_id = $2)\n END END AS digest_source\n FROM lineage ORDER BY depth", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id!", + "type_info": "Uuid" + }, + { + "ordinal": 1, + "name": "parent_job", + "type_info": "Uuid" + }, + { + "ordinal": 2, + "name": "kind!: JobKind", + "type_info": { + "Custom": { + "name": "job_kind", + "kind": { + "Enum": [ + "script", + "preview", + "flow", + "dependencies", + "flowpreview", + "script_hub", + "identity", + "flowdependencies", + "http", + "graphql", + "postgresql", + "noop", + "appdependencies", + "deploymentcallback", + "singlestepflow", + "flowscript", + "flownode", + "appscript", + "aiagent", + "unassigned_script", + "unassigned_flow", + "unassigned_singlestepflow" + ] + } + } + } + }, + { + "ordinal": 3, + "name": "runnable_path", + "type_info": "Varchar" + }, + { + "ordinal": 4, + "name": "runnable_id", + "type_info": "Int8" + }, + { + "ordinal": 5, + "name": "trigger_kind", + "type_info": "Text" + }, + { + "ordinal": 6, + "name": "trigger", + "type_info": "Varchar" + }, + { + "ordinal": 7, + "name": "tag!", + "type_info": "Varchar" + }, + { + "ordinal": 8, + "name": "permissioned_as!", + "type_info": "Varchar" + }, + { + "ordinal": 9, + "name": "origin_verified!", + "type_info": "Bool" + }, + { + "ordinal": 10, + "name": "current_version!", + "type_info": "Bool" + }, + { + "ordinal": 11, + "name": "restarted!", + "type_info": "Bool" + }, + { + "ordinal": 12, + "name": "restarted_from", + "type_info": "Text" + }, + { + "ordinal": 13, + "name": "app_stamped!", + "type_info": "Bool" + }, + { + "ordinal": 14, + "name": "args_modules!", + "type_info": "Bool" + }, + { + "ordinal": 15, + "name": "worker_group", + "type_info": "Varchar" + }, + { + "ordinal": 16, + "name": "run_as_type", + "type_info": "Text" + }, + { + "ordinal": 17, + "name": "digest_source", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Text" + ] + }, + "nullable": [ + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ] + }, + "hash": "76440a1c5b40a897078817cc960247270c9044b363b71496ba36e1357c63a2a4" +} diff --git a/backend/.sqlx/query-77e9cb16c25defc2423621a64f020e1e07f7e50df82fb6c33f79e6faa5b77bab.json b/backend/.sqlx/query-77e9cb16c25defc2423621a64f020e1e07f7e50df82fb6c33f79e6faa5b77bab.json new file mode 100644 index 0000000000..57822becd2 --- /dev/null +++ b/backend/.sqlx/query-77e9cb16c25defc2423621a64f020e1e07f7e50df82fb6c33f79e6faa5b77bab.json @@ -0,0 +1,29 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT coalesce(flow_version_lite.value, flow_version.value) as \"value!: sqlx::types::Json>\", flow.job_token_scopes FROM flow\n LEFT JOIN flow_version\n ON flow_version.id = flow.versions[array_upper(flow.versions, 1)]\n LEFT JOIN flow_version_lite\n ON flow_version_lite.id = flow_version.id\n WHERE flow.path = $1 AND flow.workspace_id = $2 LIMIT 1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "value!: sqlx::types::Json>", + "type_info": "Jsonb" + }, + { + "ordinal": 1, + "name": "job_token_scopes", + "type_info": "TextArray" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + null, + true + ] + }, + "hash": "77e9cb16c25defc2423621a64f020e1e07f7e50df82fb6c33f79e6faa5b77bab" +} diff --git a/backend/.sqlx/query-7963eff5f7109874b94c0940eaef7d4b418a9e82be56dfe7dd65d160e0dcfd13.json b/backend/.sqlx/query-7963eff5f7109874b94c0940eaef7d4b418a9e82be56dfe7dd65d160e0dcfd13.json new file mode 100644 index 0000000000..1c8dfcfa57 --- /dev/null +++ b/backend/.sqlx/query-7963eff5f7109874b94c0940eaef7d4b418a9e82be56dfe7dd65d160e0dcfd13.json @@ -0,0 +1,24 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT wp.draining FROM v2_job j\n JOIN v2_job_queue q ON q.id = j.id\n JOIN worker_ping wp ON wp.worker = q.worker\n WHERE j.id = $1 AND j.workspace_id = $2 AND q.running AND q.started_at IS NOT NULL\n AND ($3::text[] IS NULL OR j.tag = ANY($3))", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "draining", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Text", + "TextArray" + ] + }, + "nullable": [ + false + ] + }, + "hash": "7963eff5f7109874b94c0940eaef7d4b418a9e82be56dfe7dd65d160e0dcfd13" +} diff --git a/backend/.sqlx/query-1debd472c9ffd2fc78877484f93db51f9aabed54f9894eda8ad610053ad76ce6.json b/backend/.sqlx/query-83e9c830ee816d4e4e7193a67823cab7c9a4461d3bfbdc0fe7dbdd28ae2de924.json similarity index 56% rename from backend/.sqlx/query-1debd472c9ffd2fc78877484f93db51f9aabed54f9894eda8ad610053ad76ce6.json rename to backend/.sqlx/query-83e9c830ee816d4e4e7193a67823cab7c9a4461d3bfbdc0fe7dbdd28ae2de924.json index 96147d531e..b6a5d2a2d0 100644 --- a/backend/.sqlx/query-1debd472c9ffd2fc78877484f93db51f9aabed54f9894eda8ad610053ad76ce6.json +++ b/backend/.sqlx/query-83e9c830ee816d4e4e7193a67823cab7c9a4461d3bfbdc0fe7dbdd28ae2de924.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT restart_unless_cancelled, timeout FROM script WHERE hash = $1 AND workspace_id = $2", + "query": "SELECT restart_unless_cancelled, timeout, job_token_scopes FROM script WHERE hash = $1 AND workspace_id = $2", "describe": { "columns": [ { @@ -12,6 +12,11 @@ "ordinal": 1, "name": "timeout", "type_info": "Int4" + }, + { + "ordinal": 2, + "name": "job_token_scopes", + "type_info": "TextArray" } ], "parameters": { @@ -21,9 +26,10 @@ ] }, "nullable": [ + true, true, true ] }, - "hash": "1debd472c9ffd2fc78877484f93db51f9aabed54f9894eda8ad610053ad76ce6" + "hash": "83e9c830ee816d4e4e7193a67823cab7c9a4461d3bfbdc0fe7dbdd28ae2de924" } diff --git a/backend/.sqlx/query-51f18c817d4014e847486f5a8dbf205ffa0e53d5bd3fdb97777e988389b1d69e.json b/backend/.sqlx/query-8511e096394edb1433f6572d50b66d8558866af53d829d83fa634f12a90733c0.json similarity index 61% rename from backend/.sqlx/query-51f18c817d4014e847486f5a8dbf205ffa0e53d5bd3fdb97777e988389b1d69e.json rename to backend/.sqlx/query-8511e096394edb1433f6572d50b66d8558866af53d829d83fa634f12a90733c0.json index 6eceb86132..3420b8c337 100644 --- a/backend/.sqlx/query-51f18c817d4014e847486f5a8dbf205ffa0e53d5bd3fdb97777e988389b1d69e.json +++ b/backend/.sqlx/query-8511e096394edb1433f6572d50b66d8558866af53d829d83fa634f12a90733c0.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by, schema, summary, description, lock)\n VALUES ('test-workspace', $1, 'f/test/success_script', 'export function main() { return \"ok\"; }', 'deno', 'script', 'test-user', '{}', '', '', '')\n ", + "query": "\n INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by, schema, summary, description, lock)\n VALUES ('test-workspace', $1, 'f/test/success_script', 'export function main() { return \"ok\"; }', 'bun', 'script', 'test-user', '{}', '', '', E'{}\\n//bun.lock\\n')\n ", "describe": { "columns": [], "parameters": { @@ -10,5 +10,5 @@ }, "nullable": [] }, - "hash": "51f18c817d4014e847486f5a8dbf205ffa0e53d5bd3fdb97777e988389b1d69e" + "hash": "8511e096394edb1433f6572d50b66d8558866af53d829d83fa634f12a90733c0" } diff --git a/backend/.sqlx/query-8645e610aa35088376d0919132459bce3ad17df617855e5fcf838f6349e68c97.json b/backend/.sqlx/query-8645e610aa35088376d0919132459bce3ad17df617855e5fcf838f6349e68c97.json new file mode 100644 index 0000000000..446e9da6df --- /dev/null +++ b/backend/.sqlx/query-8645e610aa35088376d0919132459bce3ad17df617855e5fcf838f6349e68c97.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO global_settings (name, value) VALUES ($1, 'true'::jsonb) ON CONFLICT (name) DO NOTHING", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "8645e610aa35088376d0919132459bce3ad17df617855e5fcf838f6349e68c97" +} diff --git a/backend/.sqlx/query-7904ac35748b1de2c7ff764f22787436d0fe74f99f298e5c399233f0d247f2ec.json b/backend/.sqlx/query-8aef4f9d27ec6df07cd671bcf90bf378e9fac4986cd29f69c1f2b8a83ba515c3.json similarity index 54% rename from backend/.sqlx/query-7904ac35748b1de2c7ff764f22787436d0fe74f99f298e5c399233f0d247f2ec.json rename to backend/.sqlx/query-8aef4f9d27ec6df07cd671bcf90bf378e9fac4986cd29f69c1f2b8a83ba515c3.json index e103a91c1e..53a9daaf9a 100644 --- a/backend/.sqlx/query-7904ac35748b1de2c7ff764f22787436d0fe74f99f298e5c399233f0d247f2ec.json +++ b/backend/.sqlx/query-8aef4f9d27ec6df07cd671bcf90bf378e9fac4986cd29f69c1f2b8a83ba515c3.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "WITH RECURSIVE chain(id, parent_job) AS (\n SELECT id, parent_job FROM v2_job WHERE id = $1 AND workspace_id = $2\n UNION ALL\n SELECT j.id, j.parent_job FROM v2_job j\n JOIN chain c ON j.id = c.parent_job AND j.workspace_id = $2\n )\n SELECT\n CASE WHEN a.agent THEN regexp_replace(j.runnable_path, '\\.chat$', '')\n ELSE j.runnable_path END AS runnable_path,\n CASE\n WHEN a.agent THEN 'agents'\n WHEN j.kind IN ('script', 'script_hub', 'unassigned_script') THEN 'scripts'\n WHEN j.kind IN ('flow', 'unassigned_flow') THEN 'flows'\n WHEN j.kind IN ('singlestepflow', 'unassigned_singlestepflow') THEN\n CASE WHEN COALESCE(\n (SELECT m->'value'->>'type'\n FROM jsonb_array_elements(j.raw_flow->'modules') m\n WHERE m->>'id' IN ('a', 'main')\n LIMIT 1),\n 'script'\n ) = 'flow' THEN 'flows' ELSE 'scripts' END\n END AS scope_kind,\n CASE WHEN j.trigger_kind = 'app' THEN j.trigger END AS launched_by_app\n FROM v2_job j JOIN chain c ON c.id = j.id,\n LATERAL (SELECT j.kind = 'flowpreview'\n AND j.raw_flow->'modules'->1 IS NULL\n AND j.raw_flow->'modules'->0->>'id' = '__wm_agent_root' AS agent) a\n WHERE j.workspace_id = $2", + "query": "WITH RECURSIVE chain(id, parent_job) AS (\n SELECT id, parent_job FROM v2_job WHERE id = $1 AND workspace_id = $2\n UNION ALL\n SELECT j.id, j.parent_job FROM v2_job j\n JOIN chain c ON j.id = c.parent_job AND j.workspace_id = $2\n )\n SELECT\n CASE WHEN a.agent THEN regexp_replace(j.runnable_path, '\\.chat$', '')\n ELSE j.runnable_path END AS runnable_path,\n CASE\n WHEN a.agent THEN 'agents'\n WHEN j.kind IN ('script', 'script_hub', 'unassigned_script') THEN 'scripts'\n WHEN j.kind IN ('flow', 'unassigned_flow') THEN 'flows'\n WHEN j.kind IN ('dependencies', 'flowdependencies') THEN 'dependencies'\n WHEN j.kind IN ('singlestepflow', 'unassigned_singlestepflow') THEN\n CASE WHEN COALESCE(\n (SELECT m->'value'->>'type'\n FROM jsonb_array_elements(j.raw_flow->'modules') m\n WHERE m->>'id' IN ('a', 'main')\n LIMIT 1),\n 'script'\n ) = 'flow' THEN 'flows' ELSE 'scripts' END\n END AS scope_kind,\n CASE WHEN j.trigger_kind = 'app' THEN j.trigger END AS launched_by_app\n FROM v2_job j JOIN chain c ON c.id = j.id,\n LATERAL (SELECT j.kind = 'flowpreview'\n AND j.raw_flow->'modules'->1 IS NULL\n AND j.raw_flow->'modules'->0->>'id' = '__wm_agent_root' AS agent) a\n WHERE j.workspace_id = $2", "describe": { "columns": [ { @@ -31,5 +31,5 @@ null ] }, - "hash": "7904ac35748b1de2c7ff764f22787436d0fe74f99f298e5c399233f0d247f2ec" + "hash": "8aef4f9d27ec6df07cd671bcf90bf378e9fac4986cd29f69c1f2b8a83ba515c3" } diff --git a/backend/.sqlx/query-1860c102a5309a8e1bb7288d0616f48e4330af7fbe61426e999c6073f630d88f.json b/backend/.sqlx/query-8f5091626054a43be07768f30cdc99bd68f1de8064ace4155f4da2cd45d1b403.json similarity index 56% rename from backend/.sqlx/query-1860c102a5309a8e1bb7288d0616f48e4330af7fbe61426e999c6073f630d88f.json rename to backend/.sqlx/query-8f5091626054a43be07768f30cdc99bd68f1de8064ace4155f4da2cd45d1b403.json index d67ddf711b..574156394d 100644 --- a/backend/.sqlx/query-1860c102a5309a8e1bb7288d0616f48e4330af7fbe61426e999c6073f630d88f.json +++ b/backend/.sqlx/query-8f5091626054a43be07768f30cdc99bd68f1de8064ace4155f4da2cd45d1b403.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT DISTINCT ON (path)\n path,\n value as \"value!: sqlx::types::Json>\",\n created_at,\n typ::text as \"typ!\"\n FROM draft\n WHERE workspace_id = $1\n AND typ IN ('app', 'raw_app')\n AND (email = $2 OR email IS NULL)\n AND NOT EXISTS (\n SELECT 1 FROM app a\n WHERE a.workspace_id = draft.workspace_id\n AND a.path = draft.path\n )\n ORDER BY path, (email IS NULL), created_at DESC", + "query": "SELECT DISTINCT ON (path)\n path,\n value as \"value!: sqlx::types::Json>\",\n created_at,\n email IS NULL as \"legacy!\"\n FROM draft\n WHERE workspace_id = $1\n AND typ = 'flow'\n AND (email = $2 OR email IS NULL)\n AND NOT EXISTS (\n SELECT 1 FROM flow f\n WHERE f.workspace_id = draft.workspace_id\n AND f.path = draft.path\n )\n ORDER BY path, (email IS NULL)", "describe": { "columns": [ { @@ -20,8 +20,8 @@ }, { "ordinal": 3, - "name": "typ!", - "type_info": "Text" + "name": "legacy!", + "type_info": "Bool" } ], "parameters": { @@ -37,5 +37,5 @@ null ] }, - "hash": "1860c102a5309a8e1bb7288d0616f48e4330af7fbe61426e999c6073f630d88f" + "hash": "8f5091626054a43be07768f30cdc99bd68f1de8064ace4155f4da2cd45d1b403" } diff --git a/backend/.sqlx/query-91fdd565bd210f8aa4c6d5f5e309573aefaeb5dbc6ac07f5e664bc8fe2cb2683.json b/backend/.sqlx/query-91fdd565bd210f8aa4c6d5f5e309573aefaeb5dbc6ac07f5e664bc8fe2cb2683.json new file mode 100644 index 0000000000..ba917c82db --- /dev/null +++ b/backend/.sqlx/query-91fdd565bd210f8aa4c6d5f5e309573aefaeb5dbc6ac07f5e664bc8fe2cb2683.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT COALESCE(config->'paused' = 'true'::jsonb, false) FROM config WHERE name = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "coalesce", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "91fdd565bd210f8aa4c6d5f5e309573aefaeb5dbc6ac07f5e664bc8fe2cb2683" +} diff --git a/backend/.sqlx/query-9212902d4eb2834ab3b429968a7a85c53c49e4ef0f42c146324d5c71fbd1e21d.json b/backend/.sqlx/query-9212902d4eb2834ab3b429968a7a85c53c49e4ef0f42c146324d5c71fbd1e21d.json deleted file mode 100644 index f105dd7825..0000000000 --- a/backend/.sqlx/query-9212902d4eb2834ab3b429968a7a85c53c49e4ef0f42c146324d5c71fbd1e21d.json +++ /dev/null @@ -1,31 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "WITH deleted AS (\n DELETE FROM v2_job_queue WHERE id = $1\n RETURNING id, workspace_id, started_at, worker, canceled_by, canceled_reason\n ), completed AS (\n INSERT INTO v2_job_completed AS cj\n ( workspace_id\n , id\n , started_at\n , duration_ms\n , result\n , result_columns\n , canceled_by\n , canceled_reason\n , flow_status\n , workflow_as_code_status\n , memory_peak\n , status\n , worker\n )\n SELECT d.workspace_id, d.id, d.started_at,\n COALESCE($9::bigint, (EXTRACT('epoch' FROM (now())) - EXTRACT('epoch' FROM (COALESCE(d.started_at, now()))))*1000),\n $3::text::jsonb, $10,\n CASE WHEN $4::BOOL THEN $5 ELSE d.canceled_by END,\n CASE WHEN $4::BOOL THEN $6 WHEN d.canceled_by IS NOT NULL THEN d.canceled_reason END,\n s.flow_status, s.workflow_as_code_status, $8,\n CASE WHEN $4::BOOL OR d.canceled_by IS NOT NULL THEN 'canceled'::job_status\n WHEN $7::BOOL THEN 'skipped'::job_status\n WHEN $2::BOOL THEN 'success'::job_status\n ELSE 'failure'::job_status END,\n d.worker\n FROM deleted d LEFT JOIN v2_job_status s ON s.id = d.id\n ON CONFLICT (id) DO UPDATE SET status = EXCLUDED.status, result = $3::text::jsonb,\n canceled_by = CASE WHEN NOT $4::BOOL AND EXCLUDED.canceled_by IS NOT NULL\n THEN EXCLUDED.canceled_by ELSE cj.canceled_by END,\n canceled_reason = CASE WHEN NOT $4::BOOL AND EXCLUDED.canceled_by IS NOT NULL\n THEN EXCLUDED.canceled_reason ELSE cj.canceled_reason END\n RETURNING duration_ms\n )\n SELECT duration_ms AS \"duration_ms!\" FROM completed", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "duration_ms!", - "type_info": "Int8" - } - ], - "parameters": { - "Left": [ - "Uuid", - "Bool", - "Text", - "Bool", - "Varchar", - "Text", - "Bool", - "Int4", - "Int8", - "TextArray" - ] - }, - "nullable": [ - false - ] - }, - "hash": "9212902d4eb2834ab3b429968a7a85c53c49e4ef0f42c146324d5c71fbd1e21d" -} diff --git a/backend/.sqlx/query-ee769c78ec0a782de18911cdcb655aad93cf23321cd0f42f7ffa5348bfbea26e.json b/backend/.sqlx/query-966b065bd6bbbff9e0ca15bee504f9cfe5e3a75427767de41e5b85c797f7d58a.json similarity index 57% rename from backend/.sqlx/query-ee769c78ec0a782de18911cdcb655aad93cf23321cd0f42f7ffa5348bfbea26e.json rename to backend/.sqlx/query-966b065bd6bbbff9e0ca15bee504f9cfe5e3a75427767de41e5b85c797f7d58a.json index bb7e4fa36b..b3a2ea6a05 100644 --- a/backend/.sqlx/query-ee769c78ec0a782de18911cdcb655aad93cf23321cd0f42f7ffa5348bfbea26e.json +++ b/backend/.sqlx/query-966b065bd6bbbff9e0ca15bee504f9cfe5e3a75427767de41e5b85c797f7d58a.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT EXISTS (SELECT 1 FROM v2_job_queue WHERE id = $1 AND workspace_id = $2 AND running)", + "query": "SELECT EXISTS (SELECT 1 FROM global_settings WHERE name = $1)", "describe": { "columns": [ { @@ -11,7 +11,6 @@ ], "parameters": { "Left": [ - "Uuid", "Text" ] }, @@ -19,5 +18,5 @@ null ] }, - "hash": "ee769c78ec0a782de18911cdcb655aad93cf23321cd0f42f7ffa5348bfbea26e" + "hash": "966b065bd6bbbff9e0ca15bee504f9cfe5e3a75427767de41e5b85c797f7d58a" } diff --git a/backend/.sqlx/query-975158ae72ec12f3f933fba1d9623bf1c925feebe746299cd1f0eda3f8b949d5.json b/backend/.sqlx/query-975158ae72ec12f3f933fba1d9623bf1c925feebe746299cd1f0eda3f8b949d5.json deleted file mode 100644 index 0b166b32a4..0000000000 --- a/backend/.sqlx/query-975158ae72ec12f3f933fba1d9623bf1c925feebe746299cd1f0eda3f8b949d5.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "WITH deleted AS (\n DELETE FROM v2_job_queue WHERE id = $1\n RETURNING id, workspace_id, started_at, worker, canceled_by, canceled_reason\n ), completed AS (\n INSERT INTO v2_job_completed AS cj\n ( workspace_id\n , id\n , started_at\n , duration_ms\n , result\n , result_columns\n , canceled_by\n , canceled_reason\n , flow_status\n , workflow_as_code_status\n , memory_peak\n , status\n , worker\n )\n SELECT d.workspace_id, d.id, d.started_at,\n COALESCE($9::bigint, (EXTRACT('epoch' FROM (now())) - EXTRACT('epoch' FROM (COALESCE(d.started_at, now()))))*1000),\n $3::text::jsonb, $10,\n CASE WHEN $4::BOOL THEN $5 ELSE d.canceled_by END,\n CASE WHEN $4::BOOL THEN $6 WHEN d.canceled_by IS NOT NULL THEN d.canceled_reason END,\n s.flow_status, s.workflow_as_code_status, $8,\n CASE WHEN $4::BOOL OR d.canceled_by IS NOT NULL THEN 'canceled'::job_status\n WHEN $7::BOOL THEN 'skipped'::job_status\n WHEN $2::BOOL THEN 'success'::job_status\n ELSE 'failure'::job_status END,\n d.worker\n FROM deleted d LEFT JOIN v2_job_status s ON s.id = d.id\n ON CONFLICT (id) DO UPDATE SET status = EXCLUDED.status, result = $3::text::jsonb,\n canceled_by = CASE WHEN NOT $4::BOOL AND EXCLUDED.canceled_by IS NOT NULL\n THEN EXCLUDED.canceled_by ELSE cj.canceled_by END,\n canceled_reason = CASE WHEN NOT $4::BOOL AND EXCLUDED.canceled_by IS NOT NULL\n THEN EXCLUDED.canceled_reason ELSE cj.canceled_reason END\n RETURNING duration_ms\n ), parent_ping AS (\n UPDATE v2_job_runtime r SET ping = now()\n FROM v2_job_queue q\n WHERE r.id = $11 AND q.id = r.id AND q.workspace_id = $12\n AND EXISTS (SELECT 1 FROM completed)\n )\n SELECT duration_ms AS \"duration_ms!\" FROM completed", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "duration_ms!", - "type_info": "Int8" - } - ], - "parameters": { - "Left": [ - "Uuid", - "Bool", - "Text", - "Bool", - "Varchar", - "Text", - "Bool", - "Int4", - "Int8", - "TextArray", - "Uuid", - "Text" - ] - }, - "nullable": [ - false - ] - }, - "hash": "975158ae72ec12f3f933fba1d9623bf1c925feebe746299cd1f0eda3f8b949d5" -} diff --git a/backend/.sqlx/query-97d8a9835344b8c72d59afc99d76b681c5120ef29eb09b8626a3fbc43a32cc0e.json b/backend/.sqlx/query-97d8a9835344b8c72d59afc99d76b681c5120ef29eb09b8626a3fbc43a32cc0e.json new file mode 100644 index 0000000000..abf8b56d79 --- /dev/null +++ b/backend/.sqlx/query-97d8a9835344b8c72d59afc99d76b681c5120ef29eb09b8626a3fbc43a32cc0e.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM runnable_job_stats WHERE hour < $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int8" + ] + }, + "nullable": [] + }, + "hash": "97d8a9835344b8c72d59afc99d76b681c5120ef29eb09b8626a3fbc43a32cc0e" +} diff --git a/backend/.sqlx/query-5223412e366b22d953867425ed6156a66727e7a469abad1664ee03a3da31078a.json b/backend/.sqlx/query-9a29bd07a66f31f7414b79cafd8073194c6990eedad4b5b540134dd13d72abb1.json similarity index 52% rename from backend/.sqlx/query-5223412e366b22d953867425ed6156a66727e7a469abad1664ee03a3da31078a.json rename to backend/.sqlx/query-9a29bd07a66f31f7414b79cafd8073194c6990eedad4b5b540134dd13d72abb1.json index be5e26ef9b..e8f288f4a4 100644 --- a/backend/.sqlx/query-5223412e366b22d953867425ed6156a66727e7a469abad1664ee03a3da31078a.json +++ b/backend/.sqlx/query-9a29bd07a66f31f7414b79cafd8073194c6990eedad4b5b540134dd13d72abb1.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT DISTINCT ON (path)\n path,\n value as \"value!: sqlx::types::Json>\",\n created_at\n FROM draft\n WHERE workspace_id = $1\n AND typ = 'flow'\n AND (email = $2 OR email IS NULL)\n AND NOT EXISTS (\n SELECT 1 FROM flow f\n WHERE f.workspace_id = draft.workspace_id\n AND f.path = draft.path\n )\n ORDER BY path, (email IS NULL)", + "query": "SELECT DISTINCT ON (path)\n path,\n value as \"value!: sqlx::types::Json>\",\n created_at,\n email IS NULL as \"legacy!\"\n FROM draft\n WHERE workspace_id = $1\n AND typ = 'script'\n AND (email = $2 OR email IS NULL)\n AND NOT EXISTS (\n SELECT 1 FROM script s\n WHERE s.workspace_id = draft.workspace_id\n AND s.path = draft.path\n )\n ORDER BY path, (email IS NULL)", "describe": { "columns": [ { @@ -17,6 +17,11 @@ "ordinal": 2, "name": "created_at", "type_info": "Timestamptz" + }, + { + "ordinal": 3, + "name": "legacy!", + "type_info": "Bool" } ], "parameters": { @@ -28,8 +33,9 @@ "nullable": [ false, false, - false + false, + null ] }, - "hash": "5223412e366b22d953867425ed6156a66727e7a469abad1664ee03a3da31078a" + "hash": "9a29bd07a66f31f7414b79cafd8073194c6990eedad4b5b540134dd13d72abb1" } diff --git a/backend/.sqlx/query-9e7a5e1ebba42217092971dd340cd9182a4812b14735e502958eaafeb4c6b33d.json b/backend/.sqlx/query-9e7a5e1ebba42217092971dd340cd9182a4812b14735e502958eaafeb4c6b33d.json new file mode 100644 index 0000000000..59ed0e1767 --- /dev/null +++ b/backend/.sqlx/query-9e7a5e1ebba42217092971dd340cd9182a4812b14735e502958eaafeb4c6b33d.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE v2_job_queue\n SET suspend = $3, suspend_until = now() + make_interval(secs => $4), started_at = null,\n extras = CASE WHEN started_at IS NULL OR extras ? 'wac_first_started_at' THEN extras\n ELSE jsonb_set(COALESCE(extras, '{}'::jsonb), '{wac_first_started_at}', to_jsonb(started_at))\n END\n WHERE id = $1 AND workspace_id = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Uuid", + "Text", + "Int4", + "Float8" + ] + }, + "nullable": [] + }, + "hash": "9e7a5e1ebba42217092971dd340cd9182a4812b14735e502958eaafeb4c6b33d" +} diff --git a/backend/.sqlx/query-a2f6ca89faaa8d8739f67cd1aae571de793b377dd6058065f51b12c974e5b665.json b/backend/.sqlx/query-a2f6ca89faaa8d8739f67cd1aae571de793b377dd6058065f51b12c974e5b665.json deleted file mode 100644 index 2358af807f..0000000000 --- a/backend/.sqlx/query-a2f6ca89faaa8d8739f67cd1aae571de793b377dd6058065f51b12c974e5b665.json +++ /dev/null @@ -1,28 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n SELECT usr.email, usage.executions\n FROM usr, LATERAL (\n SELECT COALESCE(SUM(c.duration_ms + 1000)/1000 , 0)::BIGINT executions\n FROM v2_job_completed c JOIN v2_job j USING (id)\n WHERE j.workspace_id = $1\n AND j.kind NOT IN ('flow', 'flowpreview', 'flownode', 'singlestepflow')\n AND j.permissioned_as_email = usr.email\n AND now() - '1 week'::interval < j.created_at\n ) usage\n WHERE workspace_id = $1\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "email", - "type_info": "Varchar" - }, - { - "ordinal": 1, - "name": "executions", - "type_info": "Int8" - } - ], - "parameters": { - "Left": [ - "Text" - ] - }, - "nullable": [ - false, - null - ] - }, - "hash": "a2f6ca89faaa8d8739f67cd1aae571de793b377dd6058065f51b12c974e5b665" -} diff --git a/backend/.sqlx/query-a6d355b3121f199f3827c05aaa7c8f780ab4c9d3d373d3cd0bb489e0d173040c.json b/backend/.sqlx/query-a6d355b3121f199f3827c05aaa7c8f780ab4c9d3d373d3cd0bb489e0d173040c.json new file mode 100644 index 0000000000..e1657e156b --- /dev/null +++ b/backend/.sqlx/query-a6d355b3121f199f3827c05aaa7c8f780ab4c9d3d373d3cd0bb489e0d173040c.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT COALESCE(config->'paused' = 'true'::jsonb, false) FROM config WHERE name = $1 FOR UPDATE", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "coalesce", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "a6d355b3121f199f3827c05aaa7c8f780ab4c9d3d373d3cd0bb489e0d173040c" +} diff --git a/backend/.sqlx/query-12329c3359a7944ab5fa3aa27ddca1b26f340ccf574b9fa07641fe88b2d2987c.json b/backend/.sqlx/query-a6dafb34be6088886b078ed2b554de8c72e312ef8abd5ed29482a59c8ec51dd7.json similarity index 68% rename from backend/.sqlx/query-12329c3359a7944ab5fa3aa27ddca1b26f340ccf574b9fa07641fe88b2d2987c.json rename to backend/.sqlx/query-a6dafb34be6088886b078ed2b554de8c72e312ef8abd5ed29482a59c8ec51dd7.json index 4c120699aa..bde8b28723 100644 --- a/backend/.sqlx/query-12329c3359a7944ab5fa3aa27ddca1b26f340ccf574b9fa07641fe88b2d2987c.json +++ b/backend/.sqlx/query-a6dafb34be6088886b078ed2b554de8c72e312ef8abd5ed29482a59c8ec51dd7.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "INSERT INTO flow_conversation_message (conversation_id, message_type, content, job_id, step_name, success, tool_arguments, tool_result, reasoning, attachments)\n VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)", + "query": "INSERT INTO flow_conversation_message (conversation_id, message_type, content, job_id, turn_job_id, step_name, success, tool_arguments, tool_result, reasoning, attachments)\n VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11)", "describe": { "columns": [], "parameters": { @@ -20,6 +20,7 @@ }, "Text", "Uuid", + "Uuid", "Varchar", "Bool", "Text", @@ -30,5 +31,5 @@ }, "nullable": [] }, - "hash": "12329c3359a7944ab5fa3aa27ddca1b26f340ccf574b9fa07641fe88b2d2987c" + "hash": "a6dafb34be6088886b078ed2b554de8c72e312ef8abd5ed29482a59c8ec51dd7" } diff --git a/backend/.sqlx/query-df8586283178b0e684b9a0e359efb16e38fd68a5fe2b01cd77f74a725bbf37a3.json b/backend/.sqlx/query-aed606f8c8d91a924ac457b4eabdf391947d5fc3da90ef98a7b793c2430eb7bc.json similarity index 76% rename from backend/.sqlx/query-df8586283178b0e684b9a0e359efb16e38fd68a5fe2b01cd77f74a725bbf37a3.json rename to backend/.sqlx/query-aed606f8c8d91a924ac457b4eabdf391947d5fc3da90ef98a7b793c2430eb7bc.json index 3383951c13..0bd5d0eca5 100644 --- a/backend/.sqlx/query-df8586283178b0e684b9a0e359efb16e38fd68a5fe2b01cd77f74a725bbf37a3.json +++ b/backend/.sqlx/query-aed606f8c8d91a924ac457b4eabdf391947d5fc3da90ef98a7b793c2430eb7bc.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT q.id AS \"id!\", j.created_by, j.permissioned_as, j.permissioned_as_email, j.trigger, j.trigger_kind AS \"trigger_kind: TriggerKindLabel\", j.preprocessed, j.args AS \"args: sqlx::types::Json>>\" FROM v2_job_queue q JOIN v2_job j USING (id) JOIN script s ON s.workspace_id = j.workspace_id AND s.hash = j.runnable_id WHERE j.workspace_id = $1 AND j.runnable_path = $2 AND j.kind = 'script' AND j.flow_step_id IS NULL AND j.runnable_id != $3 AND q.canceled_by IS NULL AND s.restart_unless_cancelled", + "query": "SELECT q.id AS \"id!\", j.created_by, j.permissioned_as, j.permissioned_as_email, j.trigger, j.trigger_kind AS \"trigger_kind: TriggerKindLabel\", j.preprocessed, j.args AS \"args: sqlx::types::Json>>\", p.job_token_scopes AS \"job_token_scopes?\" FROM v2_job_queue q JOIN v2_job j USING (id) JOIN script s ON s.workspace_id = j.workspace_id AND s.hash = j.runnable_id LEFT JOIN job_perms p ON p.job_id = q.id WHERE j.workspace_id = $1 AND j.runnable_path = $2 AND j.kind = 'script' AND j.flow_step_id IS NULL AND j.runnable_id != $3 AND q.canceled_by IS NULL AND s.restart_unless_cancelled", "describe": { "columns": [ { @@ -71,6 +71,11 @@ "ordinal": 7, "name": "args: sqlx::types::Json>>", "type_info": "Jsonb" + }, + { + "ordinal": 8, + "name": "job_token_scopes?", + "type_info": "TextArray" } ], "parameters": { @@ -88,8 +93,9 @@ true, true, true, + true, true ] }, - "hash": "df8586283178b0e684b9a0e359efb16e38fd68a5fe2b01cd77f74a725bbf37a3" + "hash": "aed606f8c8d91a924ac457b4eabdf391947d5fc3da90ef98a7b793c2430eb7bc" } diff --git a/backend/.sqlx/query-d2a9a9af4d00c1c05276c14e6f32044437800a4b0c60aa364ab2f362a4fd9a2c.json b/backend/.sqlx/query-b0c10d17ed684af4dc79db7a199d4438ea003b5762178f25ff12ef23c1a8f46a.json similarity index 61% rename from backend/.sqlx/query-d2a9a9af4d00c1c05276c14e6f32044437800a4b0c60aa364ab2f362a4fd9a2c.json rename to backend/.sqlx/query-b0c10d17ed684af4dc79db7a199d4438ea003b5762178f25ff12ef23c1a8f46a.json index ec3ae54f17..a5a8c00494 100644 --- a/backend/.sqlx/query-d2a9a9af4d00c1c05276c14e6f32044437800a4b0c60aa364ab2f362a4fd9a2c.json +++ b/backend/.sqlx/query-b0c10d17ed684af4dc79db7a199d4438ea003b5762178f25ff12ef23c1a8f46a.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by, schema, summary, description, lock)\n VALUES ('test-workspace', $1, 'f/test/failing_script', $2, 'deno', 'script', 'test-user', '{}', 'Failing test script', 'A script that always fails', '')\n ", + "query": "\n INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by, schema, summary, description, lock)\n VALUES ('test-workspace', $1, 'f/test/failing_script', $2, 'bun', 'script', 'test-user', '{}', 'Failing test script', 'A script that always fails', E'{}\\n//bun.lock\\n')\n ", "describe": { "columns": [], "parameters": { @@ -11,5 +11,5 @@ }, "nullable": [] }, - "hash": "d2a9a9af4d00c1c05276c14e6f32044437800a4b0c60aa364ab2f362a4fd9a2c" + "hash": "b0c10d17ed684af4dc79db7a199d4438ea003b5762178f25ff12ef23c1a8f46a" } diff --git a/backend/.sqlx/query-b159cc3ab6fddcfb561c90d20bee2ff605a1c3721a4cbcc080f98fbb87277c37.json b/backend/.sqlx/query-b159cc3ab6fddcfb561c90d20bee2ff605a1c3721a4cbcc080f98fbb87277c37.json new file mode 100644 index 0000000000..d65f75636d --- /dev/null +++ b/backend/.sqlx/query-b159cc3ab6fddcfb561c90d20bee2ff605a1c3721a4cbcc080f98fbb87277c37.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT EXISTS(SELECT 1 FROM script WHERE workspace_id = $1 AND hash = $2)", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "exists", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text", + "Int8" + ] + }, + "nullable": [ + null + ] + }, + "hash": "b159cc3ab6fddcfb561c90d20bee2ff605a1c3721a4cbcc080f98fbb87277c37" +} diff --git a/backend/.sqlx/query-b40c9e2d637cf0f2f49fd4c720b64eb6af24e8d8aa99b846f21f2ed49c320bdc.json b/backend/.sqlx/query-b40c9e2d637cf0f2f49fd4c720b64eb6af24e8d8aa99b846f21f2ed49c320bdc.json new file mode 100644 index 0000000000..db729b97b1 --- /dev/null +++ b/backend/.sqlx/query-b40c9e2d637cf0f2f49fd4c720b64eb6af24e8d8aa99b846f21f2ed49c320bdc.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO flow\n (workspace_id, path, summary, description, archived, extra_perms, dependency_job, tag, ws_error_handler_muted, dedicated_worker, timeout, visible_to_runner_only, on_behalf_of, on_behalf_of_email, concurrency_key, versions, value, schema, edited_by, edited_at, job_token_scopes)\n SELECT workspace_id, REGEXP_REPLACE(path,'u/' || $2 || '/(.*)','u/' || $1 || '/\\1'), summary, description, archived, extra_perms, dependency_job, tag, ws_error_handler_muted, dedicated_worker, timeout, visible_to_runner_only, on_behalf_of, on_behalf_of_email, concurrency_key, versions, value, schema, edited_by, edited_at, job_token_scopes\n FROM flow\n WHERE path LIKE ('u/' || $2 || '/%') AND workspace_id = $3", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "b40c9e2d637cf0f2f49fd4c720b64eb6af24e8d8aa99b846f21f2ed49c320bdc" +} diff --git a/backend/.sqlx/query-b48afe9913423a90955011fc6368b8517561f5b55a30e44801d6a9123a762584.json b/backend/.sqlx/query-b48afe9913423a90955011fc6368b8517561f5b55a30e44801d6a9123a762584.json new file mode 100644 index 0000000000..47dcca381c --- /dev/null +++ b/backend/.sqlx/query-b48afe9913423a90955011fc6368b8517561f5b55a30e44801d6a9123a762584.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT job_token_scopes FROM script WHERE path = $1 AND workspace_id = $2 AND deleted = false ORDER BY created_at DESC LIMIT 1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "job_token_scopes", + "type_info": "TextArray" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + true + ] + }, + "hash": "b48afe9913423a90955011fc6368b8517561f5b55a30e44801d6a9123a762584" +} diff --git a/backend/.sqlx/query-bfd01122256e922b00acf0e86593007c6aeccf76bfd07384ffaccf50a24d13d4.json b/backend/.sqlx/query-bfd01122256e922b00acf0e86593007c6aeccf76bfd07384ffaccf50a24d13d4.json new file mode 100644 index 0000000000..b1f969f343 --- /dev/null +++ b/backend/.sqlx/query-bfd01122256e922b00acf0e86593007c6aeccf76bfd07384ffaccf50a24d13d4.json @@ -0,0 +1,102 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO script (workspace_id, hash, path, parent_hashes, summary, description, content, created_by, schema, is_template, extra_perms, lock, language, kind, tag, envs, concurrent_limit, concurrency_time_window_s, cache_ttl, dedicated_worker, ws_error_handler_muted, priority, restart_unless_cancelled, delete_after_use, delete_after_secs, timeout, concurrency_key, visible_to_runner_only, auto_kind, codebase, has_preprocessor, schema_validation, assets, debounce_key, debounce_delay_s, cache_ignore_s3_path, runnable_settings_handle, modules, labels, on_behalf_of, on_behalf_of_email, job_token_scopes) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9::text::json, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26, $27, $28, $29, $30, $31, $32, $33, $34, $35, $36, $37, $38, $39, $40, $41, $42)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Int8", + "Varchar", + "Int8Array", + "Text", + "Text", + "Text", + "Varchar", + "Text", + "Bool", + "Jsonb", + "Text", + { + "Custom": { + "name": "script_lang", + "kind": { + "Enum": [ + "python3", + "deno", + "go", + "bash", + "postgresql", + "nativets", + "bun", + "mysql", + "bigquery", + "snowflake", + "graphql", + "powershell", + "mssql", + "php", + "bunnative", + "rust", + "ansible", + "csharp", + "oracledb", + "nu", + "java", + "duckdb", + "ruby", + "rlang", + "dbt" + ] + } + } + }, + { + "Custom": { + "name": "script_kind", + "kind": { + "Enum": [ + "script", + "trigger", + "failure", + "command", + "approval", + "preprocessor" + ] + } + } + }, + "Varchar", + "VarcharArray", + "Int4", + "Int4", + "Int4", + "Bool", + "Bool", + "Int2", + "Bool", + "Bool", + "Int4", + "Int4", + "Varchar", + "Bool", + "Varchar", + "Varchar", + "Bool", + "Bool", + "Jsonb", + "Varchar", + "Int4", + "Bool", + "Int8", + "Jsonb", + "TextArray", + "Varchar", + "Text", + "TextArray" + ] + }, + "nullable": [] + }, + "hash": "bfd01122256e922b00acf0e86593007c6aeccf76bfd07384ffaccf50a24d13d4" +} diff --git a/backend/.sqlx/query-c3c51c3536eaa6a5e702d306344db63d19f1c9ab0366e01c0092aa653ab51877.json b/backend/.sqlx/query-c3c51c3536eaa6a5e702d306344db63d19f1c9ab0366e01c0092aa653ab51877.json new file mode 100644 index 0000000000..be4fbb73c9 --- /dev/null +++ b/backend/.sqlx/query-c3c51c3536eaa6a5e702d306344db63d19f1c9ab0366e01c0092aa653ab51877.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM token WHERE email = $1 AND label = 'session' AND token_hash != $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "c3c51c3536eaa6a5e702d306344db63d19f1c9ab0366e01c0092aa653ab51877" +} diff --git a/backend/.sqlx/query-c4eaa6e3b1d30a72ef1b54b2b7f28d84f1ca3c91de86d520dce06ac12d557bd5.json b/backend/.sqlx/query-c4eaa6e3b1d30a72ef1b54b2b7f28d84f1ca3c91de86d520dce06ac12d557bd5.json new file mode 100644 index 0000000000..151296d652 --- /dev/null +++ b/backend/.sqlx/query-c4eaa6e3b1d30a72ef1b54b2b7f28d84f1ca3c91de86d520dce06ac12d557bd5.json @@ -0,0 +1,24 @@ +{ + "db_name": "PostgreSQL", + "query": "WITH inserted AS (\n INSERT INTO flow\n (workspace_id, path, summary, description, archived, extra_perms, dependency_job, tag, ws_error_handler_muted, dedicated_worker, timeout, visible_to_runner_only, on_behalf_of, on_behalf_of_email, concurrency_key, versions, value, schema, edited_by, edited_at, labels, lock_error_logs, job_token_scopes)\n SELECT workspace_id, REGEXP_REPLACE(path, 'u/' || $2 || '/(.*)', $1 || '/\\1'), summary, description, archived, extra_perms, dependency_job, tag, ws_error_handler_muted, dedicated_worker, timeout, visible_to_runner_only, on_behalf_of, on_behalf_of_email, concurrency_key, versions, value, schema, edited_by, edited_at, labels, lock_error_logs, job_token_scopes\n FROM flow\n WHERE path LIKE ('u/' || $2 || '/%') AND workspace_id = $3\n RETURNING 1\n ) SELECT COUNT(*) FROM inserted", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "count", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "c4eaa6e3b1d30a72ef1b54b2b7f28d84f1ca3c91de86d520dce06ac12d557bd5" +} diff --git a/backend/.sqlx/query-c5971326bdbfaa7dff74689d1365db4f26c41fa56b326dbd02e604c793a13477.json b/backend/.sqlx/query-c5971326bdbfaa7dff74689d1365db4f26c41fa56b326dbd02e604c793a13477.json new file mode 100644 index 0000000000..d84578d37c --- /dev/null +++ b/backend/.sqlx/query-c5971326bdbfaa7dff74689d1365db4f26c41fa56b326dbd02e604c793a13477.json @@ -0,0 +1,74 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT workspace_id, runnable_path, worker_group,\n SUM(job_count)::bigint AS \"job_count!\",\n SUM(total_duration_ms)::bigint AS \"total_duration_ms!\",\n MAX(max_memory_peak) AS \"max_memory_peak!\",\n SUM(sum_memory_peak)::bigint AS \"sum_memory_peak!\",\n SUM(memory_sample_count)::bigint AS \"memory_sample_count!\",\n SUM(total_cpu_ms)::bigint AS \"total_cpu_ms!\"\n FROM runnable_job_stats\n WHERE hour >= $1\n AND ($2::text IS NULL OR worker_group = $2)\n AND ($3::text IS NULL OR workspace_id = $3)\n GROUP BY workspace_id, runnable_path, worker_group\n ORDER BY CASE $4::text\n WHEN 'max_memory' THEN MAX(max_memory_peak)::numeric\n WHEN 'cpu' THEN SUM(total_cpu_ms)\n ELSE SUM(total_duration_ms)\n END DESC, workspace_id, runnable_path, worker_group\n LIMIT $5\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "workspace_id", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "runnable_path", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "worker_group", + "type_info": "Text" + }, + { + "ordinal": 3, + "name": "job_count!", + "type_info": "Int8" + }, + { + "ordinal": 4, + "name": "total_duration_ms!", + "type_info": "Int8" + }, + { + "ordinal": 5, + "name": "max_memory_peak!", + "type_info": "Int4" + }, + { + "ordinal": 6, + "name": "sum_memory_peak!", + "type_info": "Int8" + }, + { + "ordinal": 7, + "name": "memory_sample_count!", + "type_info": "Int8" + }, + { + "ordinal": 8, + "name": "total_cpu_ms!", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Int8", + "Text", + "Text", + "Text", + "Int8" + ] + }, + "nullable": [ + false, + false, + false, + null, + null, + null, + null, + null, + null + ] + }, + "hash": "c5971326bdbfaa7dff74689d1365db4f26c41fa56b326dbd02e604c793a13477" +} diff --git a/backend/.sqlx/query-c748617e060bc41b5922df4b433f20b6971b993e0671b6ccb45db5ef028550bc.json b/backend/.sqlx/query-c748617e060bc41b5922df4b433f20b6971b993e0671b6ccb45db5ef028550bc.json deleted file mode 100644 index 7f6fe7a81c..0000000000 --- a/backend/.sqlx/query-c748617e060bc41b5922df4b433f20b6971b993e0671b6ccb45db5ef028550bc.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "INSERT INTO worker_ping (worker_instance, worker, ip, custom_tags, worker_group, dedicated_worker, dedicated_workers, wm_version, vcpus, memory, job_isolation, native_mode) VALUES ($1, $2, COALESCE($3, 'NO IP'), $4, $5, $6, $7, $8, $9, $10, $11, $12) ON CONFLICT (worker)\n DO UPDATE set ping_at = now(), worker_instance = EXCLUDED.worker_instance, ip = COALESCE($3, worker_ping.ip), custom_tags = EXCLUDED.custom_tags, worker_group = EXCLUDED.worker_group, dedicated_worker = EXCLUDED.dedicated_worker, dedicated_workers = EXCLUDED.dedicated_workers, wm_version = EXCLUDED.wm_version, vcpus = COALESCE(EXCLUDED.vcpus, worker_ping.vcpus), memory = COALESCE(EXCLUDED.memory, worker_ping.memory), job_isolation = EXCLUDED.job_isolation, native_mode = EXCLUDED.native_mode, current_job_id = NULL, current_job_workspace_id = NULL\n RETURNING jobs_executed", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "jobs_executed", - "type_info": "Int4" - } - ], - "parameters": { - "Left": [ - "Varchar", - "Varchar", - "Text", - "TextArray", - "Varchar", - "Varchar", - "TextArray", - "Varchar", - "Int8", - "Int8", - "Text", - "Bool" - ] - }, - "nullable": [ - false - ] - }, - "hash": "c748617e060bc41b5922df4b433f20b6971b993e0671b6ccb45db5ef028550bc" -} diff --git a/backend/.sqlx/query-c921b1a03e3181bbac351545941affc87853ff2d97f567a8817260cfef43e00e.json b/backend/.sqlx/query-c921b1a03e3181bbac351545941affc87853ff2d97f567a8817260cfef43e00e.json new file mode 100644 index 0000000000..393c88720d --- /dev/null +++ b/backend/.sqlx/query-c921b1a03e3181bbac351545941affc87853ff2d97f567a8817260cfef43e00e.json @@ -0,0 +1,65 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT email, username, is_admin, is_operator, groups, folders, end_user_email,\n job_token_scopes\n FROM job_perms WHERE job_id = $1 AND workspace_id = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "email", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "username", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "is_admin", + "type_info": "Bool" + }, + { + "ordinal": 3, + "name": "is_operator", + "type_info": "Bool" + }, + { + "ordinal": 4, + "name": "groups", + "type_info": "TextArray" + }, + { + "ordinal": 5, + "name": "folders", + "type_info": "JsonbArray" + }, + { + "ordinal": 6, + "name": "end_user_email", + "type_info": "Varchar" + }, + { + "ordinal": 7, + "name": "job_token_scopes", + "type_info": "TextArray" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Text" + ] + }, + "nullable": [ + false, + false, + false, + false, + false, + false, + true, + true + ] + }, + "hash": "c921b1a03e3181bbac351545941affc87853ff2d97f567a8817260cfef43e00e" +} diff --git a/backend/.sqlx/query-cdc594b30855e9384481f561b2c16200d4e6dacb1eb274513c8f1795235dfe20.json b/backend/.sqlx/query-cc719da4da98b6fe9612a4cc2f42dd19834a3e0d0be593ea6bc1239fe5453a74.json similarity index 69% rename from backend/.sqlx/query-cdc594b30855e9384481f561b2c16200d4e6dacb1eb274513c8f1795235dfe20.json rename to backend/.sqlx/query-cc719da4da98b6fe9612a4cc2f42dd19834a3e0d0be593ea6bc1239fe5453a74.json index a68e5f1253..91cacf4123 100644 --- a/backend/.sqlx/query-cdc594b30855e9384481f561b2c16200d4e6dacb1eb274513c8f1795235dfe20.json +++ b/backend/.sqlx/query-cc719da4da98b6fe9612a4cc2f42dd19834a3e0d0be593ea6bc1239fe5453a74.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by, schema, summary, description, lock)\n VALUES ('test-workspace', 5555555555, 'f/test/error_handler', 'export function main() { return \"handled\"; }', 'deno', 'script', 'test-user', '{}', '', '', '')\n ", + "query": "\n INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by, schema, summary, description, lock)\n VALUES ('test-workspace', 5555555555, 'f/test/error_handler', 'export function main() { return \"handled\"; }', 'bun', 'script', 'test-user', '{}', '', '', E'{}\\n//bun.lock\\n')\n ", "describe": { "columns": [], "parameters": { @@ -8,5 +8,5 @@ }, "nullable": [] }, - "hash": "cdc594b30855e9384481f561b2c16200d4e6dacb1eb274513c8f1795235dfe20" + "hash": "cc719da4da98b6fe9612a4cc2f42dd19834a3e0d0be593ea6bc1239fe5453a74" } diff --git a/backend/.sqlx/query-ce39fd9650500b16f7247c8173a40c7e5d2d8a51aea2d36f2bc8c6f86d53a9b8.json b/backend/.sqlx/query-ce39fd9650500b16f7247c8173a40c7e5d2d8a51aea2d36f2bc8c6f86d53a9b8.json new file mode 100644 index 0000000000..ba7a89ef73 --- /dev/null +++ b/backend/.sqlx/query-ce39fd9650500b16f7247c8173a40c7e5d2d8a51aea2d36f2bc8c6f86d53a9b8.json @@ -0,0 +1,43 @@ +{ + "db_name": "PostgreSQL", + "query": "WITH deleted AS (\n DELETE FROM v2_job_queue WHERE id = $1\n RETURNING id, workspace_id, started_at, worker, canceled_by, canceled_reason,\n (extras->>'wac_first_started_at')::timestamptz AS first_started_at,\n COALESCE($9::bigint, (EXTRACT('epoch' FROM (now())) - EXTRACT('epoch' FROM (COALESCE(started_at, now()))))*1000)::bigint AS run_ms\n ), completed AS (\n INSERT INTO v2_job_completed AS cj\n ( workspace_id\n , id\n , started_at\n , duration_ms\n , extras\n , result\n , result_columns\n , canceled_by\n , canceled_reason\n , flow_status\n , workflow_as_code_status\n , memory_peak\n , status\n , worker\n )\n SELECT d.workspace_id, d.id, COALESCE(d.first_started_at, d.started_at),\n CASE WHEN d.first_started_at IS NULL THEN d.run_ms\n ELSE EXTRACT('epoch' FROM (now() - d.first_started_at))*1000 END,\n CASE WHEN d.first_started_at IS NOT NULL\n THEN jsonb_build_object('wac_last_segment_ms', d.run_ms) END,\n $3::text::jsonb, $10,\n CASE WHEN $4::BOOL THEN $5 ELSE d.canceled_by END,\n CASE WHEN $4::BOOL THEN $6 WHEN d.canceled_by IS NOT NULL THEN d.canceled_reason END,\n s.flow_status, s.workflow_as_code_status, $8,\n CASE WHEN $4::BOOL OR d.canceled_by IS NOT NULL THEN 'canceled'::job_status\n WHEN $7::BOOL THEN 'skipped'::job_status\n WHEN $2::BOOL THEN 'success'::job_status\n ELSE 'failure'::job_status END,\n d.worker\n FROM deleted d LEFT JOIN v2_job_status s ON s.id = d.id\n ON CONFLICT (id) DO UPDATE SET status = EXCLUDED.status, result = $3::text::jsonb,\n canceled_by = CASE WHEN NOT $4::BOOL AND EXCLUDED.canceled_by IS NOT NULL\n THEN EXCLUDED.canceled_by ELSE cj.canceled_by END,\n canceled_reason = CASE WHEN NOT $4::BOOL AND EXCLUDED.canceled_by IS NOT NULL\n THEN EXCLUDED.canceled_reason ELSE cj.canceled_reason END\n RETURNING cj.started_at, cj.duration_ms,\n COALESCE((cj.extras->>'wac_last_segment_ms')::bigint, cj.duration_ms) AS last_run_ms\n )\n SELECT started_at AS \"started_at?\", duration_ms AS \"duration_ms!\",\n last_run_ms AS \"last_run_ms!\" FROM completed", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "started_at?", + "type_info": "Timestamptz" + }, + { + "ordinal": 1, + "name": "duration_ms!", + "type_info": "Int8" + }, + { + "ordinal": 2, + "name": "last_run_ms!", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Bool", + "Text", + "Bool", + "Varchar", + "Text", + "Bool", + "Int4", + "Int8", + "TextArray" + ] + }, + "nullable": [ + true, + false, + null + ] + }, + "hash": "ce39fd9650500b16f7247c8173a40c7e5d2d8a51aea2d36f2bc8c6f86d53a9b8" +} diff --git a/backend/.sqlx/query-ce3bccee14cd107631c4ddb4a467099be925f1058c76c6af31e8a620607cf6a1.json b/backend/.sqlx/query-ce3bccee14cd107631c4ddb4a467099be925f1058c76c6af31e8a620607cf6a1.json new file mode 100644 index 0000000000..4ae34431dc --- /dev/null +++ b/backend/.sqlx/query-ce3bccee14cd107631c4ddb4a467099be925f1058c76c6af31e8a620607cf6a1.json @@ -0,0 +1,30 @@ +{ + "db_name": "PostgreSQL", + "query": "\n UPDATE\n flow\n SET\n path = $1,\n summary = $2,\n description = $3,\n dependency_job = NULL,\n lock_error_logs = '',\n tag = $4,\n dedicated_worker = $5,\n visible_to_runner_only = $6,\n ws_error_handler_muted = $7,\n value = $8,\n schema = $9::text::json,\n edited_by = $10,\n edited_at = now(),\n labels = COALESCE($13, labels),\n on_behalf_of = $14,\n on_behalf_of_email = $15,\n job_token_scopes = CASE WHEN $16 THEN $17 ELSE job_token_scopes END\n WHERE\n path = $11 AND workspace_id = $12", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Text", + "Text", + "Varchar", + "Bool", + "Bool", + "Bool", + "Jsonb", + "Text", + "Varchar", + "Text", + "Text", + "TextArray", + "Varchar", + "Text", + "Bool", + "TextArray" + ] + }, + "nullable": [] + }, + "hash": "ce3bccee14cd107631c4ddb4a467099be925f1058c76c6af31e8a620607cf6a1" +} diff --git a/backend/.sqlx/query-d2fbaaaea6c542342862d8b535673d19dc4ca721ab656a39330bad5ea48999d8.json b/backend/.sqlx/query-d2fbaaaea6c542342862d8b535673d19dc4ca721ab656a39330bad5ea48999d8.json new file mode 100644 index 0000000000..55b335fba0 --- /dev/null +++ b/backend/.sqlx/query-d2fbaaaea6c542342862d8b535673d19dc4ca721ab656a39330bad5ea48999d8.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM runnable_job_stats WHERE workspace_id = $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [] + }, + "hash": "d2fbaaaea6c542342862d8b535673d19dc4ca721ab656a39330bad5ea48999d8" +} diff --git a/backend/.sqlx/query-d51f555e80b27549b884f334087878286dcc2f23101c2e1c8c1d07b6f0deaecd.json b/backend/.sqlx/query-d51f555e80b27549b884f334087878286dcc2f23101c2e1c8c1d07b6f0deaecd.json new file mode 100644 index 0000000000..ae90b163ff --- /dev/null +++ b/backend/.sqlx/query-d51f555e80b27549b884f334087878286dcc2f23101c2e1c8c1d07b6f0deaecd.json @@ -0,0 +1,159 @@ +{ + "db_name": "PostgreSQL", + "query": "select hash, tag, concurrency_key, concurrent_limit, concurrency_time_window_s, debounce_key, debounce_delay_s, cache_ttl, cache_ignore_s3_path, runnable_settings_handle, language as \"language: ScriptLang\", dedicated_worker, priority, timeout, on_behalf_of, created_by, labels, job_token_scopes FROM script\n WHERE path = $1 AND workspace_id = $2 AND archived = false AND (lock IS NOT NULL OR $3 = false)\n ORDER BY created_at DESC LIMIT 1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "hash", + "type_info": "Int8" + }, + { + "ordinal": 1, + "name": "tag", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "concurrency_key", + "type_info": "Varchar" + }, + { + "ordinal": 3, + "name": "concurrent_limit", + "type_info": "Int4" + }, + { + "ordinal": 4, + "name": "concurrency_time_window_s", + "type_info": "Int4" + }, + { + "ordinal": 5, + "name": "debounce_key", + "type_info": "Varchar" + }, + { + "ordinal": 6, + "name": "debounce_delay_s", + "type_info": "Int4" + }, + { + "ordinal": 7, + "name": "cache_ttl", + "type_info": "Int4" + }, + { + "ordinal": 8, + "name": "cache_ignore_s3_path", + "type_info": "Bool" + }, + { + "ordinal": 9, + "name": "runnable_settings_handle", + "type_info": "Int8" + }, + { + "ordinal": 10, + "name": "language: ScriptLang", + "type_info": { + "Custom": { + "name": "script_lang", + "kind": { + "Enum": [ + "python3", + "deno", + "go", + "bash", + "postgresql", + "nativets", + "bun", + "mysql", + "bigquery", + "snowflake", + "graphql", + "powershell", + "mssql", + "php", + "bunnative", + "rust", + "ansible", + "csharp", + "oracledb", + "nu", + "java", + "duckdb", + "ruby", + "rlang", + "dbt" + ] + } + } + } + }, + { + "ordinal": 11, + "name": "dedicated_worker", + "type_info": "Bool" + }, + { + "ordinal": 12, + "name": "priority", + "type_info": "Int2" + }, + { + "ordinal": 13, + "name": "timeout", + "type_info": "Int4" + }, + { + "ordinal": 14, + "name": "on_behalf_of", + "type_info": "Varchar" + }, + { + "ordinal": 15, + "name": "created_by", + "type_info": "Varchar" + }, + { + "ordinal": 16, + "name": "labels", + "type_info": "TextArray" + }, + { + "ordinal": 17, + "name": "job_token_scopes", + "type_info": "TextArray" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Bool" + ] + }, + "nullable": [ + false, + true, + true, + true, + true, + true, + true, + true, + true, + true, + false, + true, + true, + true, + true, + false, + true, + true + ] + }, + "hash": "d51f555e80b27549b884f334087878286dcc2f23101c2e1c8c1d07b6f0deaecd" +} diff --git a/backend/.sqlx/query-967f52005f4a044b3a2e9f02ceadf90dad5246681bde6caaa633b85a5e8b2352.json b/backend/.sqlx/query-d5f1e272af4a428daedafa3453124b1973a48d47c1c563b2213055bfcf688230.json similarity index 57% rename from backend/.sqlx/query-967f52005f4a044b3a2e9f02ceadf90dad5246681bde6caaa633b85a5e8b2352.json rename to backend/.sqlx/query-d5f1e272af4a428daedafa3453124b1973a48d47c1c563b2213055bfcf688230.json index 5161f716ff..190c14788f 100644 --- a/backend/.sqlx/query-967f52005f4a044b3a2e9f02ceadf90dad5246681bde6caaa633b85a5e8b2352.json +++ b/backend/.sqlx/query-d5f1e272af4a428daedafa3453124b1973a48d47c1c563b2213055bfcf688230.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "DELETE FROM flow_conversation_message WHERE job_id = ANY($1) RETURNING conversation_id", + "query": "DELETE FROM flow_conversation_message WHERE COALESCE(turn_job_id, job_id) = ANY($1)\n RETURNING conversation_id", "describe": { "columns": [ { @@ -18,5 +18,5 @@ false ] }, - "hash": "967f52005f4a044b3a2e9f02ceadf90dad5246681bde6caaa633b85a5e8b2352" + "hash": "d5f1e272af4a428daedafa3453124b1973a48d47c1c563b2213055bfcf688230" } diff --git a/backend/.sqlx/query-ebce1633afe3d3f8a1a68bc8d26adcdd88b137bb892437a3ca67fd716772512b.json b/backend/.sqlx/query-d61e8523269a26051079b0ee837bf33c1ce2e063aaf80fd155dbbbf99eb2fab3.json similarity index 59% rename from backend/.sqlx/query-ebce1633afe3d3f8a1a68bc8d26adcdd88b137bb892437a3ca67fd716772512b.json rename to backend/.sqlx/query-d61e8523269a26051079b0ee837bf33c1ce2e063aaf80fd155dbbbf99eb2fab3.json index 1930427b94..7b1e8d29cc 100644 --- a/backend/.sqlx/query-ebce1633afe3d3f8a1a68bc8d26adcdd88b137bb892437a3ca67fd716772512b.json +++ b/backend/.sqlx/query-d61e8523269a26051079b0ee837bf33c1ce2e063aaf80fd155dbbbf99eb2fab3.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT\n COUNT(*) FILTER (WHERE NOT operator AND NOT disabled AND NOT is_service_account) AS \"developers!\",\n COUNT(*) FILTER (WHERE operator AND NOT disabled AND NOT is_service_account) AS \"operators!\"\n FROM usr WHERE workspace_id = $1", + "query": "SELECT\n COUNT(*) FILTER (WHERE NOT operator AND NOT disabled AND NOT is_service_account) AS \"developers!\",\n COUNT(*) FILTER (WHERE operator AND NOT disabled AND NOT is_service_account) AS \"operators!\",\n COUNT(*) FILTER (WHERE is_service_account AND NOT disabled) AS \"service_accounts!\"\n FROM usr WHERE workspace_id = $1", "describe": { "columns": [ { @@ -12,6 +12,11 @@ "ordinal": 1, "name": "operators!", "type_info": "Int8" + }, + { + "ordinal": 2, + "name": "service_accounts!", + "type_info": "Int8" } ], "parameters": { @@ -20,9 +25,10 @@ ] }, "nullable": [ + null, null, null ] }, - "hash": "ebce1633afe3d3f8a1a68bc8d26adcdd88b137bb892437a3ca67fd716772512b" + "hash": "d61e8523269a26051079b0ee837bf33c1ce2e063aaf80fd155dbbbf99eb2fab3" } diff --git a/backend/.sqlx/query-de1adae1306fb9d877ec6f2d52235b3f57d4fa4c27861b51da6bfe8c74abd429.json b/backend/.sqlx/query-de1adae1306fb9d877ec6f2d52235b3f57d4fa4c27861b51da6bfe8c74abd429.json new file mode 100644 index 0000000000..48963d9ef7 --- /dev/null +++ b/backend/.sqlx/query-de1adae1306fb9d877ec6f2d52235b3f57d4fa4c27861b51da6bfe8c74abd429.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT q.id FROM v2_job_queue q\n JOIN native_retry_attempt n ON n.job_id = q.id\n JOIN v2_job r ON r.id = q.id\n JOIN v2_job a ON a.id = r.parent_job\n WHERE q.workspace_id = $2 AND a.parent_job = ANY($1)", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Uuid" + } + ], + "parameters": { + "Left": [ + "UuidArray", + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "de1adae1306fb9d877ec6f2d52235b3f57d4fa4c27861b51da6bfe8c74abd429" +} diff --git a/backend/.sqlx/query-df1fe2e7ef004b8de0e626f40fd737b3b4b662a89148cf6ac70eb3563ad7f0c5.json b/backend/.sqlx/query-df1fe2e7ef004b8de0e626f40fd737b3b4b662a89148cf6ac70eb3563ad7f0c5.json new file mode 100644 index 0000000000..7f160dc856 --- /dev/null +++ b/backend/.sqlx/query-df1fe2e7ef004b8de0e626f40fd737b3b4b662a89148cf6ac70eb3563ad7f0c5.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO flow (\n workspace_id, path, summary, description, value, edited_by, edited_at,\n archived, schema, extra_perms, dependency_job, tag,\n ws_error_handler_muted, dedicated_worker, timeout, visible_to_runner_only,\n concurrency_key, versions, on_behalf_of, on_behalf_of_email, lock_error_logs,\n job_token_scopes\n )\n SELECT $2, path, summary, description, value, edited_by, edited_at,\n archived, schema, extra_perms, NULL, tag,\n ws_error_handler_muted, dedicated_worker, timeout, visible_to_runner_only,\n concurrency_key, ARRAY[]::bigint[],\n -- Same predicate as clone_scripts.\n CASE WHEN on_behalf_of LIKE 'u/%' THEN\n (SELECT on_behalf_of WHERE EXISTS (\n SELECT 1 FROM usr u WHERE u.workspace_id = $2::varchar\n AND u.username = substring(on_behalf_of from 3)\n UNION ALL\n SELECT 1 FROM password p WHERE p.super_admin\n AND (p.username = substring(on_behalf_of from 3)\n OR p.email = substring(on_behalf_of from 3))))\n WHEN on_behalf_of LIKE 'g/%' THEN\n (SELECT on_behalf_of WHERE EXISTS (\n SELECT 1 FROM group_ g WHERE g.workspace_id = $2::varchar\n AND g.name = substring(on_behalf_of from 3)))\n ELSE\n (SELECT on_behalf_of WHERE EXISTS (\n SELECT 1 FROM usr u WHERE u.workspace_id = $2::varchar\n AND u.username = on_behalf_of\n UNION ALL\n SELECT 1 FROM password p WHERE p.email = on_behalf_of\n AND p.super_admin))\n END, on_behalf_of_email, lock_error_logs, job_token_scopes\n FROM flow\n WHERE workspace_id = $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "df1fe2e7ef004b8de0e626f40fd737b3b4b662a89148cf6ac70eb3563ad7f0c5" +} diff --git a/backend/.sqlx/query-e12f852fa196d16862151ab490b05e6f5c25e23b7c41ce1c0e5a83bf7b118bfd.json b/backend/.sqlx/query-e12f852fa196d16862151ab490b05e6f5c25e23b7c41ce1c0e5a83bf7b118bfd.json deleted file mode 100644 index ae893b6791..0000000000 --- a/backend/.sqlx/query-e12f852fa196d16862151ab490b05e6f5c25e23b7c41ce1c0e5a83bf7b118bfd.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "WITH prev AS (SELECT started_at FROM v2_job_queue WHERE id = $1)\n UPDATE v2_job_queue q SET running = false, started_at = null\n FROM prev WHERE q.id = $1\n RETURNING (extract(epoch FROM now() - prev.started_at) * 1000)::bigint", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "int8", - "type_info": "Int8" - } - ], - "parameters": { - "Left": [ - "Uuid" - ] - }, - "nullable": [ - null - ] - }, - "hash": "e12f852fa196d16862151ab490b05e6f5c25e23b7c41ce1c0e5a83bf7b118bfd" -} diff --git a/backend/.sqlx/query-e4b9c0e8a46dbfe29bace8d8b12123a98ec175f9d3dc72e40a1cb40e14dec919.json b/backend/.sqlx/query-e4b9c0e8a46dbfe29bace8d8b12123a98ec175f9d3dc72e40a1cb40e14dec919.json new file mode 100644 index 0000000000..9ebf9289c2 --- /dev/null +++ b/backend/.sqlx/query-e4b9c0e8a46dbfe29bace8d8b12123a98ec175f9d3dc72e40a1cb40e14dec919.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE worker_ping SET draining = true WHERE worker = $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [] + }, + "hash": "e4b9c0e8a46dbfe29bace8d8b12123a98ec175f9d3dc72e40a1cb40e14dec919" +} diff --git a/backend/.sqlx/query-e7a8c0ee68b8626eb45a7eacdd364360a5d5c783d6ca0a3c4123ea126ed0e60b.json b/backend/.sqlx/query-e7a8c0ee68b8626eb45a7eacdd364360a5d5c783d6ca0a3c4123ea126ed0e60b.json new file mode 100644 index 0000000000..9f51c69616 --- /dev/null +++ b/backend/.sqlx/query-e7a8c0ee68b8626eb45a7eacdd364360a5d5c783d6ca0a3c4123ea126ed0e60b.json @@ -0,0 +1,34 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO worker_ping (worker_instance, worker, ip, custom_tags, worker_group, dedicated_worker, dedicated_workers, wm_version, vcpus, memory, job_isolation, native_mode, draining) VALUES ($1, $2, COALESCE($3, 'NO IP'), $4, $5, $6, $7, $8, $9, $10, $11, $12, COALESCE($13, false)) ON CONFLICT (worker)\n DO UPDATE set ping_at = now(), worker_instance = EXCLUDED.worker_instance, ip = COALESCE($3, worker_ping.ip), custom_tags = EXCLUDED.custom_tags, worker_group = EXCLUDED.worker_group, dedicated_worker = EXCLUDED.dedicated_worker, dedicated_workers = EXCLUDED.dedicated_workers, wm_version = EXCLUDED.wm_version, vcpus = COALESCE(EXCLUDED.vcpus, worker_ping.vcpus), memory = COALESCE(EXCLUDED.memory, worker_ping.memory), job_isolation = EXCLUDED.job_isolation, native_mode = EXCLUDED.native_mode, current_job_id = NULL, current_job_workspace_id = NULL, draining = COALESCE($13, worker_ping.draining)\n RETURNING jobs_executed", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "jobs_executed", + "type_info": "Int4" + } + ], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Text", + "TextArray", + "Varchar", + "Varchar", + "TextArray", + "Varchar", + "Int8", + "Int8", + "Text", + "Bool", + "Bool" + ] + }, + "nullable": [ + false + ] + }, + "hash": "e7a8c0ee68b8626eb45a7eacdd364360a5d5c783d6ca0a3c4123ea126ed0e60b" +} diff --git a/backend/.sqlx/query-e81eaa501b4af1269a4929ec660d10fc10d9ec4dc2359f43d79fe255d101cad8.json b/backend/.sqlx/query-e81eaa501b4af1269a4929ec660d10fc10d9ec4dc2359f43d79fe255d101cad8.json new file mode 100644 index 0000000000..909565b8c6 --- /dev/null +++ b/backend/.sqlx/query-e81eaa501b4af1269a4929ec660d10fc10d9ec4dc2359f43d79fe255d101cad8.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO flow\n (workspace_id, path, summary, description, archived, extra_perms, dependency_job, tag, ws_error_handler_muted, dedicated_worker, timeout, visible_to_runner_only, on_behalf_of, on_behalf_of_email, concurrency_key, versions, value, schema, edited_by, edited_at, lock_error_logs, job_token_scopes)\n SELECT $1, path, summary, description, archived, extra_perms, dependency_job, tag, ws_error_handler_muted, dedicated_worker, timeout, visible_to_runner_only, on_behalf_of, on_behalf_of_email, concurrency_key, versions, value, schema, edited_by, edited_at, lock_error_logs, job_token_scopes\n FROM flow WHERE workspace_id = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Text" + ] + }, + "nullable": [] + }, + "hash": "e81eaa501b4af1269a4929ec660d10fc10d9ec4dc2359f43d79fe255d101cad8" +} diff --git a/backend/.sqlx/query-e8779f1cddc475554dbbd2e3122db8a21a2eaacf971bf046cef1ee14cf04e323.json b/backend/.sqlx/query-e8779f1cddc475554dbbd2e3122db8a21a2eaacf971bf046cef1ee14cf04e323.json new file mode 100644 index 0000000000..e22953610d --- /dev/null +++ b/backend/.sqlx/query-e8779f1cddc475554dbbd2e3122db8a21a2eaacf971bf046cef1ee14cf04e323.json @@ -0,0 +1,28 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT usr.email, usage.executions\n FROM usr, LATERAL (\n SELECT COALESCE(SUM(COALESCE((c.extras->>'wac_last_segment_ms')::bigint, c.duration_ms) + 1000)/1000 , 0)::BIGINT executions\n FROM v2_job_completed c JOIN v2_job j USING (id)\n WHERE j.workspace_id = $1\n AND j.kind NOT IN ('flow', 'flowpreview', 'flownode', 'singlestepflow')\n AND j.permissioned_as_email = usr.email\n AND now() - '1 week'::interval < j.created_at\n ) usage\n WHERE workspace_id = $1\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "email", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "executions", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false, + null + ] + }, + "hash": "e8779f1cddc475554dbbd2e3122db8a21a2eaacf971bf046cef1ee14cf04e323" +} diff --git a/backend/.sqlx/query-ee92100fcd681515ba81f23970e64af7f33464879bef4d1eff682126dffeef64.json b/backend/.sqlx/query-ee92100fcd681515ba81f23970e64af7f33464879bef4d1eff682126dffeef64.json deleted file mode 100644 index 3c52f8ba96..0000000000 --- a/backend/.sqlx/query-ee92100fcd681515ba81f23970e64af7f33464879bef4d1eff682126dffeef64.json +++ /dev/null @@ -1,125 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "WITH RECURSIVE lineage AS (\n SELECT id, parent_job, kind, runnable_path, runnable_id, trigger_kind, trigger,\n permissioned_as, args, 0 AS depth\n FROM v2_job WHERE id = $1 AND workspace_id = $2\n UNION ALL\n SELECT p.id, p.parent_job, p.kind, p.runnable_path, p.runnable_id, p.trigger_kind,\n p.trigger, p.permissioned_as, p.args, l.depth + 1\n FROM v2_job p JOIN lineage l ON p.id = l.parent_job\n WHERE p.workspace_id = $2 AND l.depth < 100\n )\n SELECT id AS \"id!\", parent_job, kind AS \"kind!: JobKind\", runnable_path, runnable_id,\n trigger_kind::text AS trigger_kind, permissioned_as AS \"permissioned_as!\",\n -- A restart takes its flow version from the request, so a trusted flow path\n -- can carry another flow's code: the version must belong to that path.\n CASE kind\n WHEN 'flow' THEN EXISTS (SELECT 1 FROM flow_version fv\n WHERE fv.id = runnable_id AND fv.path = runnable_path AND fv.workspace_id = $2)\n WHEN 'script' THEN EXISTS (SELECT 1 FROM script s\n WHERE s.hash = runnable_id AND s.path = runnable_path AND s.workspace_id = $2\n AND NOT s.deleted)\n ELSE true\n END AS \"origin_verified!\",\n -- What `run/f` and `run/p` resolve the path to now; for scripts the predicate of\n -- `get_latest_deployed_script_hash`.\n COALESCE(CASE kind\n WHEN 'flow' THEN runnable_id = (SELECT f.versions[array_upper(f.versions, 1)]\n FROM flow f WHERE f.path = runnable_path AND f.workspace_id = $2)\n WHEN 'script' THEN runnable_id = (SELECT s.hash FROM script s\n WHERE s.path = runnable_path AND s.workspace_id = $2 AND NOT s.deleted\n AND s.lock IS NOT NULL AND s.lock_error_logs IS NULL\n ORDER BY s.created_at DESC LIMIT 1)\n END, false) AS \"current_version!\",\n EXISTS (SELECT 1 FROM v2_job_status st WHERE st.id = lineage.id\n AND jsonb_typeof(st.flow_status->'restarted_from') = 'object')\n OR EXISTS (SELECT 1 FROM v2_job_completed c WHERE c.id = lineage.id\n AND jsonb_typeof(c.flow_status->'restarted_from') = 'object')\n AS \"restarted!\",\n COALESCE(\n (SELECT st.flow_status->'restarted_from'->>'flow_job_id' FROM v2_job_status st\n WHERE st.id = lineage.id),\n (SELECT c.flow_status->'restarted_from'->>'flow_job_id' FROM v2_job_completed c\n WHERE c.id = lineage.id)\n ) AS restarted_from,\n -- Only deployed-app runs are stamped with their app; an app editor preview\n -- runs app code at an app path it does not have to own.\n COALESCE(trigger_kind = 'app' AND starts_with(runnable_path, trigger || '/'), false)\n AS \"app_stamped!\",\n -- A preview's modules come from its args, which its parent may have taken from\n -- the caller.\n COALESCE(jsonb_typeof(args->'_MODULES') = 'object', false) AS \"args_modules!\"\n FROM lineage ORDER BY depth", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "id!", - "type_info": "Uuid" - }, - { - "ordinal": 1, - "name": "parent_job", - "type_info": "Uuid" - }, - { - "ordinal": 2, - "name": "kind!: JobKind", - "type_info": { - "Custom": { - "name": "job_kind", - "kind": { - "Enum": [ - "script", - "preview", - "flow", - "dependencies", - "flowpreview", - "script_hub", - "identity", - "flowdependencies", - "http", - "graphql", - "postgresql", - "noop", - "appdependencies", - "deploymentcallback", - "singlestepflow", - "flowscript", - "flownode", - "appscript", - "aiagent", - "unassigned_script", - "unassigned_flow", - "unassigned_singlestepflow" - ] - } - } - } - }, - { - "ordinal": 3, - "name": "runnable_path", - "type_info": "Varchar" - }, - { - "ordinal": 4, - "name": "runnable_id", - "type_info": "Int8" - }, - { - "ordinal": 5, - "name": "trigger_kind", - "type_info": "Text" - }, - { - "ordinal": 6, - "name": "permissioned_as!", - "type_info": "Varchar" - }, - { - "ordinal": 7, - "name": "origin_verified!", - "type_info": "Bool" - }, - { - "ordinal": 8, - "name": "current_version!", - "type_info": "Bool" - }, - { - "ordinal": 9, - "name": "restarted!", - "type_info": "Bool" - }, - { - "ordinal": 10, - "name": "restarted_from", - "type_info": "Text" - }, - { - "ordinal": 11, - "name": "app_stamped!", - "type_info": "Bool" - }, - { - "ordinal": 12, - "name": "args_modules!", - "type_info": "Bool" - } - ], - "parameters": { - "Left": [ - "Uuid", - "Text" - ] - }, - "nullable": [ - null, - null, - null, - null, - null, - null, - null, - null, - null, - null, - null, - null, - null - ] - }, - "hash": "ee92100fcd681515ba81f23970e64af7f33464879bef4d1eff682126dffeef64" -} diff --git a/backend/.sqlx/query-e34cb7cd6ff9d80e77a51ae47c3cfb9b0772d5d3d239273fc5cea322cae4e258.json b/backend/.sqlx/query-eecea4a5ba0de73129bde6ed0c229690c322bc376b1bfc13ed719126f72cabb9.json similarity index 80% rename from backend/.sqlx/query-e34cb7cd6ff9d80e77a51ae47c3cfb9b0772d5d3d239273fc5cea322cae4e258.json rename to backend/.sqlx/query-eecea4a5ba0de73129bde6ed0c229690c322bc376b1bfc13ed719126f72cabb9.json index 4ebad828df..1670a5009e 100644 --- a/backend/.sqlx/query-e34cb7cd6ff9d80e77a51ae47c3cfb9b0772d5d3d239273fc5cea322cae4e258.json +++ b/backend/.sqlx/query-eecea4a5ba0de73129bde6ed0c229690c322bc376b1bfc13ed719126f72cabb9.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "INSERT INTO job_resolution (job_id, workspace_id, resolved_by, note, automatic)\n SELECT c.id, c.workspace_id, NULL, NULL, true\n FROM v2_job_completed c\n JOIN v2_job j ON j.id = c.id\n WHERE c.status = 'failure'\n AND c.workspace_id = $2\n AND j.flow_step_id IS NULL\n AND j.runnable_id IS NOT DISTINCT FROM $3\n AND EXISTS (\n SELECT 1 FROM v2_job_completed sc\n JOIN v2_job sj ON sj.id = sc.id\n JOIN native_retry_attempt nra ON nra.job_id = sc.id\n WHERE sc.status = 'success'\n AND sc.workspace_id = $2\n AND sj.parent_job = $1\n AND sj.runnable_id IS NOT DISTINCT FROM $3\n )\n AND (\n (j.parent_job = $1\n AND EXISTS (SELECT 1 FROM native_retry_attempt WHERE job_id = c.id))\n OR (c.id = $1 AND j.parent_job IS NULL\n AND NOT EXISTS (SELECT 1 FROM native_retry_attempt WHERE job_id = c.id))\n )\n ON CONFLICT (job_id) DO NOTHING", + "query": "INSERT INTO job_resolution (job_id, workspace_id, resolved_by, note, automatic)\n SELECT c.id, c.workspace_id, NULL, NULL, true\n FROM v2_job_completed c\n JOIN v2_job j ON j.id = c.id\n WHERE c.status = 'failure'\n AND c.workspace_id = $2\n AND j.flow_step_id IS NULL\n AND j.runnable_id IS NOT DISTINCT FROM $3\n AND EXISTS (\n SELECT 1 FROM v2_job_completed sc\n JOIN v2_job sj ON sj.id = sc.id\n JOIN native_retry_attempt nra ON nra.job_id = sc.id\n WHERE sc.status = 'success'\n AND sc.workspace_id = $2\n AND sj.parent_job = $1\n AND sj.runnable_id IS NOT DISTINCT FROM $3\n )\n AND (\n (j.parent_job = $1\n AND EXISTS (SELECT 1 FROM native_retry_attempt WHERE job_id = c.id))\n OR (c.id = $1\n AND NOT EXISTS (SELECT 1 FROM native_retry_attempt WHERE job_id = c.id))\n )\n ON CONFLICT (job_id) DO NOTHING", "describe": { "columns": [], "parameters": { @@ -12,5 +12,5 @@ }, "nullable": [] }, - "hash": "e34cb7cd6ff9d80e77a51ae47c3cfb9b0772d5d3d239273fc5cea322cae4e258" + "hash": "eecea4a5ba0de73129bde6ed0c229690c322bc376b1bfc13ed719126f72cabb9" } diff --git a/backend/.sqlx/query-eefb19c05e6d9650b4dd1e50e10c26f01e117b75ab48e197c7d91d388865da74.json b/backend/.sqlx/query-eefb19c05e6d9650b4dd1e50e10c26f01e117b75ab48e197c7d91d388865da74.json new file mode 100644 index 0000000000..24c87414b1 --- /dev/null +++ b/backend/.sqlx/query-eefb19c05e6d9650b4dd1e50e10c26f01e117b75ab48e197c7d91d388865da74.json @@ -0,0 +1,398 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT\n v2_job.id,\n v2_job.raw_code,\n v2_job.raw_lock,\n v2_job.raw_flow as \"raw_flow: _\",\n v2_job.tag,\n v2_job.created_at,\n v2_job.created_by,\n v2_job.permissioned_as,\n v2_job.permissioned_as_email,\n v2_job.kind as \"kind: _\",\n v2_job.runnable_id,\n v2_job.runnable_path,\n v2_job.parent_job,\n v2_job.root_job,\n v2_job.script_lang as \"script_lang: _\",\n v2_job.script_entrypoint_override,\n v2_job.flow_step,\n v2_job.flow_step_id,\n v2_job.flow_innermost_root_job,\n v2_job.\"trigger\",\n v2_job.trigger_kind as \"trigger_kind: _\",\n v2_job.same_worker,\n v2_job.visible_to_owner,\n v2_job.concurrent_limit,\n v2_job.concurrency_time_window_s,\n v2_job.cache_ttl,\n v2_job.timeout,\n v2_job.priority,\n v2_job.preprocessed,\n v2_job.args as \"args: _\",\n v2_job.labels,\n job_perms.job_token_scopes as \"job_token_scopes?\",\n v2_job.pre_run_error,\n\n v2_job_queue.started_at,\n v2_job_queue.scheduled_for,\n v2_job_queue.running,\n v2_job_queue.canceled_by,\n v2_job_queue.canceled_reason,\n v2_job_queue.suspend,\n v2_job_queue.suspend_until,\n v2_job_queue.worker,\n v2_job_queue.extras as \"extras: _\",\n\n v2_job_runtime.ping,\n v2_job_runtime.memory_peak,\n\n v2_job_status.flow_status as \"flow_status: _\",\n v2_job_status.flow_leaf_jobs as \"flow_leaf_jobs: _\",\n v2_job_status.workflow_as_code_status as \"workflow_as_code_status: _\",\n\n concurrency_key.key as \"concurrency_key?\"\n FROM v2_job_queue\n INNER JOIN v2_job ON v2_job.id = v2_job_queue.id\n LEFT JOIN job_perms ON job_perms.job_id = v2_job_queue.id\n LEFT JOIN v2_job_runtime ON v2_job_runtime.id = v2_job_queue.id\n LEFT JOIN v2_job_status ON v2_job_status.id = v2_job_queue.id\n LEFT JOIN concurrency_key ON concurrency_key.job_id = v2_job_queue.id\n WHERE v2_job_queue.workspace_id = $1\n AND v2_job_queue.running = false\n AND v2_job.parent_job IS NULL\n AND v2_job.trigger_kind IS DISTINCT FROM 'schedule'\n ORDER BY v2_job.created_at DESC\n LIMIT $2\n OFFSET $3\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Uuid" + }, + { + "ordinal": 1, + "name": "raw_code", + "type_info": "Text" + }, + { + "ordinal": 2, + "name": "raw_lock", + "type_info": "Text" + }, + { + "ordinal": 3, + "name": "raw_flow: _", + "type_info": "Jsonb" + }, + { + "ordinal": 4, + "name": "tag", + "type_info": "Varchar" + }, + { + "ordinal": 5, + "name": "created_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 6, + "name": "created_by", + "type_info": "Varchar" + }, + { + "ordinal": 7, + "name": "permissioned_as", + "type_info": "Varchar" + }, + { + "ordinal": 8, + "name": "permissioned_as_email", + "type_info": "Varchar" + }, + { + "ordinal": 9, + "name": "kind: _", + "type_info": { + "Custom": { + "name": "job_kind", + "kind": { + "Enum": [ + "script", + "preview", + "flow", + "dependencies", + "flowpreview", + "script_hub", + "identity", + "flowdependencies", + "http", + "graphql", + "postgresql", + "noop", + "appdependencies", + "deploymentcallback", + "singlestepflow", + "flowscript", + "flownode", + "appscript", + "aiagent", + "unassigned_script", + "unassigned_flow", + "unassigned_singlestepflow" + ] + } + } + } + }, + { + "ordinal": 10, + "name": "runnable_id", + "type_info": "Int8" + }, + { + "ordinal": 11, + "name": "runnable_path", + "type_info": "Varchar" + }, + { + "ordinal": 12, + "name": "parent_job", + "type_info": "Uuid" + }, + { + "ordinal": 13, + "name": "root_job", + "type_info": "Uuid" + }, + { + "ordinal": 14, + "name": "script_lang: _", + "type_info": { + "Custom": { + "name": "script_lang", + "kind": { + "Enum": [ + "python3", + "deno", + "go", + "bash", + "postgresql", + "nativets", + "bun", + "mysql", + "bigquery", + "snowflake", + "graphql", + "powershell", + "mssql", + "php", + "bunnative", + "rust", + "ansible", + "csharp", + "oracledb", + "nu", + "java", + "duckdb", + "ruby", + "rlang", + "dbt" + ] + } + } + } + }, + { + "ordinal": 15, + "name": "script_entrypoint_override", + "type_info": "Varchar" + }, + { + "ordinal": 16, + "name": "flow_step", + "type_info": "Int4" + }, + { + "ordinal": 17, + "name": "flow_step_id", + "type_info": "Varchar" + }, + { + "ordinal": 18, + "name": "flow_innermost_root_job", + "type_info": "Uuid" + }, + { + "ordinal": 19, + "name": "trigger", + "type_info": "Varchar" + }, + { + "ordinal": 20, + "name": "trigger_kind: _", + "type_info": { + "Custom": { + "name": "job_trigger_kind", + "kind": { + "Enum": [ + "webhook", + "http", + "websocket", + "kafka", + "email", + "nats", + "schedule", + "app", + "ui", + "postgres", + "sqs", + "gcp", + "mqtt", + "nextcloud", + "google", + "ci_test", + "github", + "azure", + "asset", + "freshness", + "amqp" + ] + } + } + } + }, + { + "ordinal": 21, + "name": "same_worker", + "type_info": "Bool" + }, + { + "ordinal": 22, + "name": "visible_to_owner", + "type_info": "Bool" + }, + { + "ordinal": 23, + "name": "concurrent_limit", + "type_info": "Int4" + }, + { + "ordinal": 24, + "name": "concurrency_time_window_s", + "type_info": "Int4" + }, + { + "ordinal": 25, + "name": "cache_ttl", + "type_info": "Int4" + }, + { + "ordinal": 26, + "name": "timeout", + "type_info": "Int4" + }, + { + "ordinal": 27, + "name": "priority", + "type_info": "Int2" + }, + { + "ordinal": 28, + "name": "preprocessed", + "type_info": "Bool" + }, + { + "ordinal": 29, + "name": "args: _", + "type_info": "Jsonb" + }, + { + "ordinal": 30, + "name": "labels", + "type_info": "TextArray" + }, + { + "ordinal": 31, + "name": "job_token_scopes?", + "type_info": "TextArray" + }, + { + "ordinal": 32, + "name": "pre_run_error", + "type_info": "Text" + }, + { + "ordinal": 33, + "name": "started_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 34, + "name": "scheduled_for", + "type_info": "Timestamptz" + }, + { + "ordinal": 35, + "name": "running", + "type_info": "Bool" + }, + { + "ordinal": 36, + "name": "canceled_by", + "type_info": "Varchar" + }, + { + "ordinal": 37, + "name": "canceled_reason", + "type_info": "Text" + }, + { + "ordinal": 38, + "name": "suspend", + "type_info": "Int4" + }, + { + "ordinal": 39, + "name": "suspend_until", + "type_info": "Timestamptz" + }, + { + "ordinal": 40, + "name": "worker", + "type_info": "Varchar" + }, + { + "ordinal": 41, + "name": "extras: _", + "type_info": "Jsonb" + }, + { + "ordinal": 42, + "name": "ping", + "type_info": "Timestamptz" + }, + { + "ordinal": 43, + "name": "memory_peak", + "type_info": "Int4" + }, + { + "ordinal": 44, + "name": "flow_status: _", + "type_info": "Jsonb" + }, + { + "ordinal": 45, + "name": "flow_leaf_jobs: _", + "type_info": "Jsonb" + }, + { + "ordinal": 46, + "name": "workflow_as_code_status: _", + "type_info": "Jsonb" + }, + { + "ordinal": 47, + "name": "concurrency_key?", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text", + "Int8", + "Int8" + ] + }, + "nullable": [ + false, + true, + true, + true, + false, + false, + false, + false, + false, + false, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + false, + false, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + false, + false, + true, + true, + false, + true, + true, + true, + true, + true, + true, + true, + true, + false + ] + }, + "hash": "eefb19c05e6d9650b4dd1e50e10c26f01e117b75ab48e197c7d91d388865da74" +} diff --git a/backend/.sqlx/query-cbc521b505c3953dc624ae6cefe6999e6d0c732da83ebc74ef7678cacd22ad6b.json b/backend/.sqlx/query-efc7aab19d729ea419c1fcaef47122096921dce2dac3979a8b788bf76f32646b.json similarity index 58% rename from backend/.sqlx/query-cbc521b505c3953dc624ae6cefe6999e6d0c732da83ebc74ef7678cacd22ad6b.json rename to backend/.sqlx/query-efc7aab19d729ea419c1fcaef47122096921dce2dac3979a8b788bf76f32646b.json index a2a87b680d..81505d64a3 100644 --- a/backend/.sqlx/query-cbc521b505c3953dc624ae6cefe6999e6d0c732da83ebc74ef7678cacd22ad6b.json +++ b/backend/.sqlx/query-efc7aab19d729ea419c1fcaef47122096921dce2dac3979a8b788bf76f32646b.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "WITH active_users as (SELECT distinct username as email FROM audit_partitioned WHERE timestamp > NOW() - INTERVAL '1 month' AND (operation = 'users.login' OR operation = 'oauth.login' OR operation = 'users.token.refresh') AND username NOT IN (SELECT email FROM usr WHERE is_service_account)),\n active_authors as (SELECT distinct t.email FROM usr t LEFT JOIN workspace_settings ws ON ws.workspace_id = t.workspace_id WHERE NOT (t.operator AND NOT COALESCE((ws.operator_settings->>'builder_flows')::boolean, false)) AND t.email IN (SELECT email FROM active_users)),\n active_authors_agg as (SELECT array_agg(email) as authors FROM active_authors),\n active_ops_agg as (SELECT array_agg(email) as operators from active_users WHERE email NOT IN (SELECT email FROM active_authors))\n SELECT active_authors_agg.authors, active_ops_agg.operators, array_length(active_authors_agg.authors, 1) as author_count, array_length(active_ops_agg.operators, 1) as operator_count FROM active_authors_agg, active_ops_agg", + "query": "WITH active_users as (SELECT distinct username as email FROM audit_partitioned WHERE timestamp > NOW() - INTERVAL '1 month' AND (operation = 'users.login' OR operation = 'oauth.login' OR operation = 'users.token.refresh') AND username NOT IN (SELECT email FROM usr WHERE is_service_account)),\n active_authors as (SELECT distinct t.email FROM usr t LEFT JOIN workspace_settings ws ON ws.workspace_id = t.workspace_id WHERE NOT (t.operator AND NOT (COALESCE((ws.operator_settings->>'builder_flows')::boolean, false) OR COALESCE((ws.operator_settings->>'builder_apps')::boolean, false))) AND t.email IN (SELECT email FROM active_users)),\n active_authors_agg as (SELECT array_agg(email) as authors FROM active_authors),\n active_ops_agg as (SELECT array_agg(email) as operators from active_users WHERE email NOT IN (SELECT email FROM active_authors))\n SELECT active_authors_agg.authors, active_ops_agg.operators, array_length(active_authors_agg.authors, 1) as author_count, array_length(active_ops_agg.operators, 1) as operator_count FROM active_authors_agg, active_ops_agg", "describe": { "columns": [ { @@ -34,5 +34,5 @@ null ] }, - "hash": "cbc521b505c3953dc624ae6cefe6999e6d0c732da83ebc74ef7678cacd22ad6b" + "hash": "efc7aab19d729ea419c1fcaef47122096921dce2dac3979a8b788bf76f32646b" } diff --git a/backend/.sqlx/query-f0498c9bddc0d4d8948167f3ed849bf8b3523d8dde1ef6b8f1950423830cc6ed.json b/backend/.sqlx/query-f0498c9bddc0d4d8948167f3ed849bf8b3523d8dde1ef6b8f1950423830cc6ed.json deleted file mode 100644 index 5f86fb97d8..0000000000 --- a/backend/.sqlx/query-f0498c9bddc0d4d8948167f3ed849bf8b3523d8dde1ef6b8f1950423830cc6ed.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE v2_job_queue\n SET suspend = $3, suspend_until = now() + make_interval(secs => $4), started_at = null\n WHERE id = $1 AND workspace_id = $2", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Uuid", - "Text", - "Int4", - "Float8" - ] - }, - "nullable": [] - }, - "hash": "f0498c9bddc0d4d8948167f3ed849bf8b3523d8dde1ef6b8f1950423830cc6ed" -} diff --git a/backend/.sqlx/query-f2146082f64fcdb4acdfe92e42f084f90b3ac06139ffca98670215956150563a.json b/backend/.sqlx/query-f2146082f64fcdb4acdfe92e42f084f90b3ac06139ffca98670215956150563a.json new file mode 100644 index 0000000000..ad4991a7cf --- /dev/null +++ b/backend/.sqlx/query-f2146082f64fcdb4acdfe92e42f084f90b3ac06139ffca98670215956150563a.json @@ -0,0 +1,28 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO flow (\n workspace_id, path, summary, description,\n dependency_job, lock_error_logs, tag,\n dedicated_worker, visible_to_runner_only,\n ws_error_handler_muted,\n value, schema, edited_by, edited_at, labels,\n on_behalf_of, on_behalf_of_email, job_token_scopes\n ) VALUES (\n $1, $2, $3, $4,\n NULL, '', $5,\n $6, $7,\n $8,\n $9, $10::text::json, $11, now(), $12,\n $13, $14, $15\n )", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Text", + "Text", + "Varchar", + "Bool", + "Bool", + "Bool", + "Jsonb", + "Text", + "Varchar", + "TextArray", + "Varchar", + "Text", + "TextArray" + ] + }, + "nullable": [] + }, + "hash": "f2146082f64fcdb4acdfe92e42f084f90b3ac06139ffca98670215956150563a" +} diff --git a/backend/.sqlx/query-f338741f0866dc717ccfcf6b7d46b4e55ca07e472610b29fad24b64f45f98e18.json b/backend/.sqlx/query-f338741f0866dc717ccfcf6b7d46b4e55ca07e472610b29fad24b64f45f98e18.json new file mode 100644 index 0000000000..0f57f5d951 --- /dev/null +++ b/backend/.sqlx/query-f338741f0866dc717ccfcf6b7d46b4e55ca07e472610b29fad24b64f45f98e18.json @@ -0,0 +1,59 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM draft\n WHERE workspace_id = $1 AND path = $2 AND typ = $3\n AND ($4::text IS NULL OR email = $4)\n RETURNING 1 as \"one!\"", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "one!", + "type_info": "Int4" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + { + "Custom": { + "name": "draft_kind", + "kind": { + "Enum": [ + "script", + "flow", + "app", + "raw_app", + "resource", + "variable", + "trigger_schedule", + "trigger_webhook", + "trigger_default_email", + "trigger_email", + "trigger_http", + "trigger_websocket", + "trigger_postgres", + "trigger_kafka", + "trigger_nats", + "trigger_mqtt", + "trigger_sqs", + "trigger_gcp", + "trigger_azure", + "trigger_poll", + "trigger_cli", + "trigger_nextcloud", + "trigger_google", + "trigger_github", + "data_pipeline", + "trigger_amqp" + ] + } + } + }, + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "f338741f0866dc717ccfcf6b7d46b4e55ca07e472610b29fad24b64f45f98e18" +} diff --git a/backend/.sqlx/query-f633095ebe2fdabd116d687c7f8cfb8cb76159d8db20a42e0fce71146f9b8aaa.json b/backend/.sqlx/query-f633095ebe2fdabd116d687c7f8cfb8cb76159d8db20a42e0fce71146f9b8aaa.json new file mode 100644 index 0000000000..c8134447e6 --- /dev/null +++ b/backend/.sqlx/query-f633095ebe2fdabd116d687c7f8cfb8cb76159d8db20a42e0fce71146f9b8aaa.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO global_settings (name, value) VALUES ($1, $2) ON CONFLICT (name) DO UPDATE SET value = EXCLUDED.value, updated_at = now() WHERE global_settings.value IS DISTINCT FROM EXCLUDED.value", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Jsonb" + ] + }, + "nullable": [] + }, + "hash": "f633095ebe2fdabd116d687c7f8cfb8cb76159d8db20a42e0fce71146f9b8aaa" +} diff --git a/backend/.sqlx/query-69bfbe9b39414b724488532cc3b3659915d9fcb3d58f16532aaffe06c44ec976.json b/backend/.sqlx/query-fa81e10f725586e12c77d4f6387cc76d5569621612ade0270261c172102015b4.json similarity index 67% rename from backend/.sqlx/query-69bfbe9b39414b724488532cc3b3659915d9fcb3d58f16532aaffe06c44ec976.json rename to backend/.sqlx/query-fa81e10f725586e12c77d4f6387cc76d5569621612ade0270261c172102015b4.json index 52105ca605..6fbc58c626 100644 --- a/backend/.sqlx/query-69bfbe9b39414b724488532cc3b3659915d9fcb3d58f16532aaffe06c44ec976.json +++ b/backend/.sqlx/query-fa81e10f725586e12c77d4f6387cc76d5569621612ade0270261c172102015b4.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "DELETE FROM flow_conversation_message m\n USING flow_conversation c\n WHERE m.conversation_id = c.id AND c.workspace_id = $1 AND m.job_id = ANY($2)\n RETURNING m.conversation_id", + "query": "DELETE FROM flow_conversation_message m\n USING flow_conversation c\n WHERE m.conversation_id = c.id AND c.workspace_id = $1\n AND COALESCE(m.turn_job_id, m.job_id) = ANY($2)\n RETURNING m.conversation_id", "describe": { "columns": [ { @@ -19,5 +19,5 @@ false ] }, - "hash": "69bfbe9b39414b724488532cc3b3659915d9fcb3d58f16532aaffe06c44ec976" + "hash": "fa81e10f725586e12c77d4f6387cc76d5569621612ade0270261c172102015b4" } diff --git a/backend/.sqlx/query-fac99e27c8396185dd21c33baf649dd75b5f8ed859e410d2322ec3f565382fa8.json b/backend/.sqlx/query-fac99e27c8396185dd21c33baf649dd75b5f8ed859e410d2322ec3f565382fa8.json deleted file mode 100644 index c624143ea4..0000000000 --- a/backend/.sqlx/query-fac99e27c8396185dd21c33baf649dd75b5f8ed859e410d2322ec3f565382fa8.json +++ /dev/null @@ -1,35 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT DISTINCT ON (path)\n path,\n value as \"value!: sqlx::types::Json>\",\n created_at\n FROM draft\n WHERE workspace_id = $1\n AND typ = 'script'\n AND (email = $2 OR email IS NULL)\n AND NOT EXISTS (\n SELECT 1 FROM script s\n WHERE s.workspace_id = draft.workspace_id\n AND s.path = draft.path\n )\n ORDER BY path, (email IS NULL)", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "path", - "type_info": "Varchar" - }, - { - "ordinal": 1, - "name": "value!: sqlx::types::Json>", - "type_info": "Json" - }, - { - "ordinal": 2, - "name": "created_at", - "type_info": "Timestamptz" - } - ], - "parameters": { - "Left": [ - "Text", - "Text" - ] - }, - "nullable": [ - false, - false, - false - ] - }, - "hash": "fac99e27c8396185dd21c33baf649dd75b5f8ed859e410d2322ec3f565382fa8" -} diff --git a/backend/.sqlx/query-68bca8f839e47705b11d312ee874eceaa3d1d24d9053ad4aea94b9f8465585ca.json b/backend/.sqlx/query-fe22c4609d8b2472c0f83fedb92eceec2b6f1ae1a32fcf375161d04419404d9c.json similarity index 86% rename from backend/.sqlx/query-68bca8f839e47705b11d312ee874eceaa3d1d24d9053ad4aea94b9f8465585ca.json rename to backend/.sqlx/query-fe22c4609d8b2472c0f83fedb92eceec2b6f1ae1a32fcf375161d04419404d9c.json index 583d2bcc6b..5ee5c7571d 100644 --- a/backend/.sqlx/query-68bca8f839e47705b11d312ee874eceaa3d1d24d9053ad4aea94b9f8465585ca.json +++ b/backend/.sqlx/query-fe22c4609d8b2472c0f83fedb92eceec2b6f1ae1a32fcf375161d04419404d9c.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT worker, worker_instance, EXTRACT(EPOCH FROM (now() - ping_at))::integer as last_ping, started_at, ip, jobs_executed,\n CASE WHEN $4 IS TRUE THEN current_job_id ELSE NULL END as last_job_id, CASE WHEN $4 IS TRUE THEN current_job_workspace_id ELSE NULL END as last_job_workspace_id,\n custom_tags, worker_group, wm_version, occupancy_rate, occupancy_rate_15s, occupancy_rate_5m, occupancy_rate_30m, memory, vcpus, memory_usage, wm_memory_usage, job_isolation, native_mode\n FROM worker_ping\n WHERE ($1::integer IS NULL AND ping_at > now() - interval '5 minute') OR (ping_at > now() - ($1 || ' seconds')::interval)\n ORDER BY ping_at desc LIMIT $2 OFFSET $3", + "query": "SELECT worker, worker_instance, EXTRACT(EPOCH FROM (now() - ping_at))::integer as last_ping, started_at, ip, jobs_executed,\n CASE WHEN $4 IS TRUE THEN current_job_id ELSE NULL END as last_job_id, CASE WHEN $4 IS TRUE THEN current_job_workspace_id ELSE NULL END as last_job_workspace_id,\n custom_tags, worker_group, wm_version, occupancy_rate, occupancy_rate_15s, occupancy_rate_5m, occupancy_rate_30m, memory, vcpus, memory_usage, wm_memory_usage, job_isolation, native_mode, draining\n FROM worker_ping\n WHERE ($1::integer IS NULL AND ping_at > now() - interval '5 minute') OR (ping_at > now() - ($1 || ' seconds')::interval)\n ORDER BY ping_at desc LIMIT $2 OFFSET $3", "describe": { "columns": [ { @@ -107,6 +107,11 @@ "ordinal": 20, "name": "native_mode", "type_info": "Bool" + }, + { + "ordinal": 21, + "name": "draining", + "type_info": "Bool" } ], "parameters": { @@ -138,8 +143,9 @@ true, true, true, + false, false ] }, - "hash": "68bca8f839e47705b11d312ee874eceaa3d1d24d9053ad4aea94b9f8465585ca" + "hash": "fe22c4609d8b2472c0f83fedb92eceec2b6f1ae1a32fcf375161d04419404d9c" } diff --git a/backend/AGENTS.md b/backend/AGENTS.md index 750e0594e1..fbacb857ee 100644 --- a/backend/AGENTS.md +++ b/backend/AGENTS.md @@ -214,7 +214,7 @@ Whole classes of defect compile and unit-test clean: - **Stack overflow from a large buffer in an async block.** An array declared across an `.await` is baked into the future's state; once that future is boxed a few layers deep by the job poller, two 16 KB arrays abort the worker *process* (`thread - 'tokio-runtime-worker' has overflowed its stack`). Heap-allocate read buffers + 'tokio-rt-worker' has overflowed its stack`). Heap-allocate read buffers (`vec![0u8; N]`, not `[0u8; N]`). - Deadlocks from draining only one of a child's pipes, missed cancellation or timeout propagation, and anything depending on the real engine's output format. diff --git a/backend/Cargo.lock b/backend/Cargo.lock index abc735f0c7..1bd38513a7 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -2,15 +2,6 @@ # It is not intended for manual editing. version = 4 -[[package]] -name = "addr2line" -version = "0.25.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b5d307320b3181d6d7954e663bd7c774a838b8220fe0593c86d9fb09f498b4b" -dependencies = [ - "gimli", -] - [[package]] name = "adler2" version = "2.0.1" @@ -247,7 +238,7 @@ version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "73cd58deff2140a0a8eae87e417bd01db68a33e148aa93d1e8cd837e55e312b6" dependencies = [ - "object 0.39.1", + "object", ] [[package]] @@ -508,6 +499,22 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d92bec98840b8f03a5ff5413de5293bfcd8bf96467cf5452609f939ec6f5de16" +[[package]] +name = "asn1-rs" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f6fd5ddaf0351dff5b8da21b2fb4ff8e08ddd02857f0bf69c47639106c0fff0" +dependencies = [ + "asn1-rs-derive 0.4.0", + "asn1-rs-impl 0.1.0", + "displaydoc", + "nom", + "num-traits", + "rusticata-macros", + "thiserror 1.0.69", + "time", +] + [[package]] name = "asn1-rs" version = "0.6.2" @@ -515,7 +522,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5493c3bedbacf7fd7382c6346bbd66687d12bbaad3a89a2d2c303ee6cf20b048" dependencies = [ "asn1-rs-derive 0.5.1", - "asn1-rs-impl", + "asn1-rs-impl 0.2.0", "displaydoc", "nom", "num-traits", @@ -531,7 +538,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b7f43a50ac4fdca5df8e885c21b835997f0a1cdee65494a6847694a98652d9d8" dependencies = [ "asn1-rs-derive 0.6.0", - "asn1-rs-impl", + "asn1-rs-impl 0.2.0", "displaydoc", "nom", "num-traits", @@ -540,6 +547,18 @@ dependencies = [ "time", ] +[[package]] +name = "asn1-rs-derive" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "726535892e8eae7e70657b4c8ea93d26b8553afb1ce617caee529ef96d7dee6c" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", + "synstructure 0.12.6", +] + [[package]] name = "asn1-rs-derive" version = "0.5.1" @@ -564,6 +583,17 @@ dependencies = [ "synstructure 0.13.2", ] +[[package]] +name = "asn1-rs-impl" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2777730b2039ac0f95f093556e61b6d26cebed5393ca6f152717777cec3a42ed" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "asn1-rs-impl" version = "0.2.0" @@ -828,13 +858,13 @@ dependencies = [ [[package]] name = "async-recursion" -version = "1.1.1" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b43422f69d8ff38f95f1b2bb76517c91589a924d1559a0e935d7c8ce0274c11" +checksum = "4f8abc12baad266b1c8cec146854c195b5864b4221d4b2ca7296a7ae82d9e451" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn 3.0.6", ] [[package]] @@ -920,6 +950,35 @@ version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" +[[package]] +name = "attribute-derive" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2f9ef7a68684657ce9da341c934c73b63df7a3585a61e366d84c84ed5d3d167" +dependencies = [ + "attribute-derive-macro", + "manyhow 0.14.0", + "proc-macro2", + "quote", + "syn 3.0.6", +] + +[[package]] +name = "attribute-derive-macro" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300d426e5e73549125bb8e4649bdd5508f82c94ddf898fb0517246e71dfb8e98" +dependencies = [ + "collection_literals", + "interpolator", + "manyhow 0.14.0", + "proc-macro-utils", + "proc-macro2", + "quote", + "quote-use", + "syn 3.0.6", +] + [[package]] name = "autocfg" version = "1.5.1" @@ -974,11 +1033,23 @@ version = "1.18.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b281d307588d634de920874890732659e2e7672f72b5e10e81badc1a8a83621e" dependencies = [ - "aws-lc-sys", + "aws-lc-sys 0.45.0", "untrusted 0.7.1", "zeroize", ] +[[package]] +name = "aws-lc-sys" +version = "0.40.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f50037ee5e1e41e7b8f9d161680a725bd1626cb6f8c7e901f91f942850852fe7" +dependencies = [ + "cc", + "cmake", + "dunce", + "fs_extra", +] + [[package]] name = "aws-lc-sys" version = "0.45.0" @@ -1254,7 +1325,7 @@ dependencies = [ "bytes", "form_urlencoded", "hex", - "hmac", + "hmac 0.12.1", "http 0.2.12", "http 1.5.0", "percent-encoding", @@ -1338,18 +1409,18 @@ dependencies = [ "aws-smithy-runtime-api", "aws-smithy-types", "h2 0.3.27", - "h2 0.4.19", + "h2 0.4.20", "http 0.2.12", "http 1.5.0", "http-body 0.4.6", "hyper 0.14.32", - "hyper 1.11.1", + "hyper 1.12.0", "hyper-rustls 0.24.2", "hyper-rustls 0.27.10", "hyper-util", "pin-project-lite", "rustls 0.21.12", - "rustls 0.23.35", + "rustls 0.23.40", "rustls-native-certs 0.8.4", "rustls-pki-types", "tokio", @@ -1492,44 +1563,17 @@ dependencies = [ "aws-smithy-async", "aws-smithy-runtime-api", "aws-smithy-types", - "rustc_version 0.4.1", + "rustc_version", "tracing", ] -[[package]] -name = "axum" -version = "0.7.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "edca88bc138befd0323b20752846e6587272d3b03b0343c8ea28a6f819e6e71f" -dependencies = [ - "async-trait", - "axum-core 0.4.5", - "bytes", - "futures-util", - "http 1.5.0", - "http-body 1.1.0", - "http-body-util", - "itoa", - "matchit 0.7.3", - "memchr", - "mime", - "percent-encoding", - "pin-project-lite", - "rustversion", - "serde", - "sync_wrapper", - "tower 0.5.3", - "tower-layer", - "tower-service", -] - [[package]] name = "axum" version = "0.8.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "31b698c5f9a010f6573133b09e0de5408834d0c82f8d7475a89fc1867a71cd90" dependencies = [ - "axum-core 0.5.6", + "axum-core", "axum-macros", "bytes", "form_urlencoded", @@ -1537,7 +1581,7 @@ dependencies = [ "http 1.5.0", "http-body 1.1.0", "http-body-util", - "hyper 1.11.1", + "hyper 1.12.0", "hyper-util", "itoa", "matchit 0.8.4", @@ -1558,26 +1602,6 @@ dependencies = [ "tracing", ] -[[package]] -name = "axum-core" -version = "0.4.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09f2bd6146b97ae3359fa0cc6d6b376d9539582c7b4220f041a33ec24c226199" -dependencies = [ - "async-trait", - "bytes", - "futures-util", - "http 1.5.0", - "http-body 1.1.0", - "http-body-util", - "mime", - "pin-project-lite", - "rustversion", - "sync_wrapper", - "tower-layer", - "tower-service", -] - [[package]] name = "axum-core" version = "0.5.6" @@ -1608,12 +1632,6 @@ dependencies = [ "syn 2.0.119", ] -[[package]] -name = "az" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be5eb007b7cacc6c660343e96f650fedf4b5a77512399eb952ca6642cf8d13f7" - [[package]] name = "backon" version = "1.6.0" @@ -1625,27 +1643,18 @@ dependencies = [ "tokio", ] -[[package]] -name = "backtrace" -version = "0.3.76" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb531853791a215d7c62a30daf0dde835f381ab5de4589cfe7c649d2cbe92bd6" -dependencies = [ - "addr2line", - "cfg-if", - "libc", - "miniz_oxide 0.8.9", - "object 0.37.3", - "rustc-demangle", - "windows-link 0.2.1", -] - [[package]] name = "base16ct" version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4c7f02d4ea65f2c1853089ffd8d2787bdbc63de2f0d29dedbcf8ccdfa0ccd4cf" +[[package]] +name = "base16ct" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd307490d624467aa6f74b0eabb77633d1f758a7b25f12bceb0b22e08d9726f6" + [[package]] name = "base32" version = "0.5.1" @@ -1723,6 +1732,26 @@ dependencies = [ "serde", ] +[[package]] +name = "bindgen" +version = "0.70.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f49d8fed880d473ea71efb9bf597651e77201bdd4893efe54c9e5d65ae04ce6f" +dependencies = [ + "bitflags 2.13.2", + "cexpr", + "clang-sys", + "itertools 0.13.0", + "log", + "prettyplease 0.2.37", + "proc-macro2", + "quote", + "regex", + "rustc-hash 1.1.0", + "shlex 1.3.0", + "syn 2.0.119", +] + [[package]] name = "bindgen" version = "0.71.1" @@ -1931,7 +1960,7 @@ dependencies = [ "hex", "http 1.5.0", "http-body-util", - "hyper 1.11.1", + "hyper 1.12.0", "hyper-named-pipe", "hyper-util", "hyperlocal", @@ -1963,18 +1992,18 @@ dependencies = [ [[package]] name = "bon" -version = "3.10.1" +version = "3.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60eafe0d77c3a2fc292c1d1346c3041b33c0a108085a2afabf672b70f69dbbc9" +checksum = "214f5df094ce551a10a30ffb9c70243d61f121a3d985a6495933e181dee6a7d2" dependencies = [ "bon-macros", ] [[package]] name = "bon-macros" -version = "3.10.1" +version = "3.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd0f9631d8aaaee112c41985d675ef269e02acbd4f33122836af4f0c5f699ff6" +checksum = "2706da6c749998cc555d04184909608956a69ad3f8ab1a076fbc867b4a3c3ce5" dependencies = [ "darling 0.24.1", "ident_case", @@ -2018,6 +2047,17 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "brotli" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74f7971dbd9326d58187408ab83117d8ac1bb9c17b085fdacd1cf2f598719b6b" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", + "brotli-decompressor 4.0.3", +] + [[package]] name = "brotli" version = "7.0.0" @@ -2212,6 +2252,16 @@ dependencies = [ "pkg-config", ] +[[package]] +name = "calendrical_calculations" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5abbd6eeda6885048d357edc66748eea6e0268e3dd11f326fff5bd248d779c26" +dependencies = [ + "core_maths", + "displaydoc", +] + [[package]] name = "candle-core" version = "0.9.2" @@ -2316,9 +2366,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.5.1" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f360145194ee8e21db5ee7f3fcd4fe52210864c75c985dae33218202c8bbe040" +checksum = "f74872d07caf508b30a21f6836e7d7016a2eaf7d9ff4f48deaa58cd8a0407630" dependencies = [ "find-msvc-tools", "jobserver", @@ -2364,6 +2414,18 @@ dependencies = [ "rand_core 0.10.1", ] +[[package]] +name = "char_str" +version = "0.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9cba78a5abb515610e10af297615434b019755b5d7be14e559ff5340dff50c4" +dependencies = [ + "castaway", + "get-size2", + "itoa", + "ryu", +] + [[package]] name = "chrono" version = "0.4.45" @@ -2489,12 +2551,18 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7b77c319abfd5219629c45c34c89ba945ed3c5e49fcde9d16b6c3885f118a730" dependencies = [ - "const-oid", - "der", - "spki", + "const-oid 0.9.6", + "der 0.7.10", + "spki 0.7.3", "x509-cert", ] +[[package]] +name = "collection_literals" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2550f75b8cfac212855f6b1885455df8eaee8fe8e246b647d69146142e016084" + [[package]] name = "colorchoice" version = "1.0.5" @@ -2534,6 +2602,19 @@ dependencies = [ "static_assertions", ] +[[package]] +name = "compact_str" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79fcda08c33bb58b97008b2cdada6622500e949e060f5913361763121abd2416" +dependencies = [ + "castaway", + "cfg-if", + "itoa", + "static_assertions", + "zmij", +] + [[package]] name = "concurrent-queue" version = "2.5.0" @@ -2562,12 +2643,30 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "const-hex" +version = "1.19.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e59eef12462b0f9b0a3620219be5d639afd79fe39dff0a42c3997061f9298b4" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "proptest", + "serde_core", +] + [[package]] name = "const-oid" version = "0.9.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" +[[package]] +name = "const-oid" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6ef517f0926dd24a1582492c791b6a4818a4d94e789a334894aa15b0d12f55c" + [[package]] name = "const-random" version = "0.1.18" @@ -2694,6 +2793,21 @@ version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" +[[package]] +name = "core_maths" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77745e017f5edba1a9c1d854f6f3a52dac8a12dd5af5d2f54aecf61e43d80d30" +dependencies = [ + "libm", +] + +[[package]] +name = "cpubits" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15b85f9c39137c3a891689859392b1bd49812121d0d61c9caf00d46ed5ce06ae" + [[package]] name = "cpufeatures" version = "0.2.17" @@ -2745,6 +2859,12 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "critical-section" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "790eea4361631c5e7d22598ecd5723ff611904e3344ce8720784c93e3d83d40b" + [[package]] name = "cron" version = "0.17.0" @@ -2836,6 +2956,22 @@ dependencies = [ "zeroize", ] +[[package]] +name = "crypto-bigint" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a52aa3fcda4e6302a9f48734f234d35d4721b96f8fe07d073f07ce9df4f0271" +dependencies = [ + "cpubits", + "ctutils", + "getrandom 0.4.3", + "hybrid-array", + "num-traits", + "rand_core 0.10.1", + "subtle", + "zeroize", +] + [[package]] name = "crypto-common" version = "0.1.7" @@ -2853,7 +2989,32 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" dependencies = [ + "getrandom 0.4.3", "hybrid-array", + "rand_core 0.10.1", +] + +[[package]] +name = "cssparser" +version = "0.36.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dae61cf9c0abb83bd659dab65b7e4e38d8236824c85f0f804f173567bda257d2" +dependencies = [ + "cssparser-macros", + "dtoa-short", + "itoa", + "phf 0.13.1", + "smallvec", +] + +[[package]] +name = "cssparser-macros" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13b588ba4ac1a99f7f2964d24b3d896ddc6bf847ee3855dbd4366f058cfcd331" +dependencies = [ + "quote", + "syn 2.0.119", ] [[package]] @@ -2888,18 +3049,19 @@ dependencies = [ [[package]] name = "ctutils" -version = "0.4.2" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d5515a3834141de9eafb9717ad39eea8247b5674e6066c404e8c4b365d2a29e" +checksum = "03bb0e1cc970d482d121d9a1744999169b69a07470b3d644a7894e53fcaf4574" dependencies = [ "cmov", + "subtle", ] [[package]] name = "curl-sys" -version = "0.4.90+curl-8.21.0" +version = "0.4.91+curl-8.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97799a0d220bfb3361e0fe4936966ff8c4b24d65c3f06dfc70d7b680b44e7897" +checksum = "cd39579123e45bf762716bb893d43886ed1cabe6289455be29a4f1d4e37afb06" dependencies = [ "cc", "libc", @@ -2920,8 +3082,8 @@ dependencies = [ "cpufeatures 0.2.17", "curve25519-dalek-derive", "digest 0.10.7", - "fiat-crypto 0.2.9", - "rustc_version 0.4.1", + "fiat-crypto", + "rustc_version", "subtle", "zeroize", ] @@ -3795,7 +3957,7 @@ checksum = "c72e0409b3dbd60a5bf296cbc273a8e36bb3a3aab6abac389f1891e187d5ce14" dependencies = [ "base64 0.22.1", "capacity_builder", - "deno_error 0.7.3", + "deno_error", "deno_media_type", "deno_terminal", "dprint-swc-ext", @@ -3829,37 +3991,29 @@ dependencies = [ "url", ] -[[package]] -name = "deno_console" -version = "0.209.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "66c0b8a65dcb7b38c22e5969c6454b3cb0839b7dcfb7a4f0d904de871a4c4416" -dependencies = [ - "deno_core", -] - [[package]] name = "deno_core" -version = "0.352.1" +version = "0.412.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf78f3f72ac8e09b18a588bc26a1b3c5ab853e7fb489889b2f5000654d34db5d" +checksum = "580cb2e0d797d75ff3dfc4a8f2b14aa7470f0fa31654a09a83d4cbb42b5eb9b4" dependencies = [ "anyhow", - "az", - "bincode", + "base64 0.22.1", "bit-set", "bit-vec 0.8.0", + "boxed_error", "bytes", - "capacity_builder", "cooked-waker", - "deno_core_icudata", - "deno_error 0.6.1", + "deno_error", "deno_ops", "deno_path_util", "deno_unsync", + "deno_v8", "futures", "indexmap 2.14.2", + "inventory", "libc", + "num-bigint", "parking_lot", "percent-encoding", "pin-project", @@ -3869,41 +4023,41 @@ dependencies = [ "smallvec", "sourcemap", "static_assertions", + "sys_traits", "thiserror 2.0.21", "tokio", "url", - "v8", + "uuid", "wasm_dep_analyzer", + "windows-sys 0.59.0", ] -[[package]] -name = "deno_core_icudata" -version = "0.74.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe4dccb6147bb3f3ba0c7a48e993bfeb999d2c2e47a81badee80e2b370c8d695" - [[package]] name = "deno_crypto" -version = "0.223.0" +version = "0.273.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "beb2568806b930b96fb6440ff16463befa73a36de3378986644e7134c9d52e8f" +checksum = "d9931d22328961b9a9c8d3b666a27312c1f3ddcc7ea78d8610ddfbc934ac61a0" dependencies = [ "aes 0.8.3", "aes-gcm", "aes-kw", + "argon2", "aws-lc-rs", "base64 0.22.1", "cbc", - "const-oid", + "const-oid 0.9.6", "ctr", "curve25519-dalek", "deno_core", - "deno_error 0.6.1", - "deno_web", + "deno_crypto_provider", + "deno_error", "ecdsa", - "ed448-goldilocks", - "elliptic-curve", + "elliptic-curve 0.13.8", + "fips203", + "fips205", + "hmac 0.12.1", "num-traits", + "ocb3", "once_cell", "p256", "p384", @@ -3914,21 +4068,34 @@ dependencies = [ "serde_bytes", "sha1", "sha2 0.10.9", - "signature", - "spki", + "sha3", + "signature 2.2.0", + "spki 0.7.3", + "subtle", "thiserror 2.0.21", + "tiny-keccak", "tokio", "uuid", "x25519-dalek", ] [[package]] -name = "deno_error" -version = "0.6.1" +name = "deno_crypto_provider" +version = "0.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "612ec3fc481fea759141b0c57810889b0a4fb6fee8f10748677bfe492fd30486" +checksum = "fafd673e5a4eefd6d65db358b00ff60488cfa1be6515406b4371935c2e2e6baa" dependencies = [ - "deno_error_macro 0.6.1", + "aws-lc-sys 0.40.0", + "ed448-goldilocks", +] + +[[package]] +name = "deno_error" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfafd2219b29886a71aecbb3449e462deed1b2c474dc5b12f855f0e58c478931" +dependencies = [ + "deno_error_macro", "libc", "serde", "serde_json", @@ -3936,32 +4103,11 @@ dependencies = [ "url", ] -[[package]] -name = "deno_error" -version = "0.7.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3007d3f1ea92ea503324ae15883aac0c2de2b8cf6fead62203ff6a67161007ab" -dependencies = [ - "deno_error_macro 0.7.3", - "libc", -] - [[package]] name = "deno_error_macro" -version = "0.6.1" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8380a4224d5d2c3f84da4d764c4326cac62e9a1e3d4960442d29136fc07be863" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.119", -] - -[[package]] -name = "deno_error_macro" -version = "0.7.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b565e60a9685cdf312c888665b5f8647ac692a7da7e058a5e2268a466da8eaf" +checksum = "1c28ede88783f14cd8aae46ca89f230c226b40e4a81ab06fa52ed72af84beb2f" dependencies = [ "proc-macro2", "quote", @@ -3970,43 +4116,41 @@ dependencies = [ [[package]] name = "deno_features" -version = "0.6.0" +version = "0.56.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "487773bb24b92f3b88d1c9ef0d4d15888641a2e2ea4d4cdfd6fda12cae26317c" +checksum = "2fef16a40daeb19acfc5345aa5bb367a36ecffef18b7ad1a594f0f01fe7c20c5" dependencies = [ "deno_core", - "serde", - "serde_json", ] [[package]] name = "deno_fetch" -version = "0.233.0" +version = "0.283.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5cbe158bec955790105ded69978b1245b97c65092f7bf184add344c0e78efffa" +checksum = "20e792f76d45e20cf6f1ad46157bbe1a0a895a6f09ec4d99b0911112be8e05e3" dependencies = [ + "async-compression", "base64 0.22.1", "bytes", "data-url", "deno_core", - "deno_error 0.6.1", + "deno_error", "deno_fs", + "deno_io", "deno_path_util", "deno_permissions", "deno_tls", - "dyn-clone", "error_reporter", - "h2 0.4.19", + "h2 0.4.20", "hickory-resolver", "http 1.5.0", "http-body-util", - "hyper 1.11.1", + "hyper 1.12.0", "hyper-rustls 0.27.10", "hyper-util", "ipnet", "percent-encoding", - "rustls-webpki 0.102.8", - "serde", + "rustls-webpki 0.103.15", "serde_json", "thiserror 2.0.21", "tokio", @@ -4015,62 +4159,69 @@ dependencies = [ "tokio-util", "tokio-vsock", "tower 0.5.3", - "tower-http", "tower-service", ] [[package]] name = "deno_fs" -version = "0.119.0" +version = "0.169.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ca83a95cea7bcdf19dac1888ef6f04aff69d251640a4d085bc6089c06b4b181" +checksum = "a0b0ae3c55f4af9d33113ba4935828a2e9a7bfcbf4d3058533403b9ef332a966" dependencies = [ "async-trait", - "base32", "boxed_error", "deno_core", - "deno_error 0.6.1", + "deno_error", "deno_io", + "deno_maybe_sync", "deno_path_util", "deno_permissions", "filetime", "junction", "libc", - "nix 0.27.1", + "nix 0.30.1", "rand 0.8.5", "rayon", + "same-file", "serde", "thiserror 2.0.21", - "winapi", + "tokio", "windows-sys 0.59.0", ] [[package]] name = "deno_io" -version = "0.119.0" +version = "0.169.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "597aac25be261a1bd545d6f47a80439c60e600c076f9847b3718a592883b2a73" +checksum = "ba495b6c141d34e49fb2dbe7ca746d7ac068b1962470776eb341c59059b758f6" dependencies = [ "async-trait", "deno_core", - "deno_error 0.6.1", + "deno_error", + "deno_permissions", "deno_subprocess_windows", "filetime", - "fs3", "libc", "log", - "nix 0.27.1", - "once_cell", + "nix 0.30.1", "os_pipe", "parking_lot", "pin-project", "rand 0.8.5", "tokio", "uuid", - "winapi", "windows-sys 0.59.0", ] +[[package]] +name = "deno_maybe_sync" +version = "0.46.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57e5e72b38015b5e478b8d739dfa818d563329e03a72d6ce5cfb2ecbcd7f8d41" +dependencies = [ + "rustc-hash 2.1.3", +] + [[package]] name = "deno_media_type" version = "0.3.3" @@ -4097,17 +4248,21 @@ dependencies = [ [[package]] name = "deno_net" -version = "0.201.0" +version = "0.251.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb88f1ea2762065d6cc0316372acc22696ab6004943aa029f80546f9faf1b491" +checksum = "19ba02152d978f30e2eb5f8d074b343363f53cb1364b5c304b3176da916a06c0" dependencies = [ "deno_core", - "deno_error 0.6.1", + "deno_error", "deno_features", "deno_permissions", + "deno_signals", "deno_tls", + "deno_tunnel", "hickory-proto", "hickory-resolver", + "libc", + "log", "pin-project", "quinn", "rustls-tokio-stream", @@ -4119,32 +4274,33 @@ dependencies = [ "tokio-vsock", "url", "web-transport-proto", + "x509-parser 0.15.1", ] [[package]] name = "deno_ops" -version = "0.228.1" +version = "0.288.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8bf8dbe5abf37d270bb853c5dfe45fbe3b1b6c453877cc11d7fe84e9862a6dbc" +checksum = "977748624aad80164dbe385c8c907bc0041c2e6849f92528e362e03132c64c52" dependencies = [ "indexmap 2.14.2", - "proc-macro-rules", "proc-macro2", "quote", "stringcase", "strum", "strum_macros", "syn 2.0.119", + "syn-match", "thiserror 2.0.21", ] [[package]] name = "deno_path_util" -version = "0.4.0" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "516f813389095889776b81cc9108ff6f336fd9409b4b12fc0138aea23d2708e1" +checksum = "78c7e98943f0d068928906db0c7bde89de684fa32c6a8018caacc4cee2cdd72b" dependencies = [ - "deno_error 0.6.1", + "deno_error", "percent-encoding", "sys_traits", "thiserror 2.0.21", @@ -4153,77 +4309,100 @@ dependencies = [ [[package]] name = "deno_permissions" -version = "0.68.0" +version = "0.118.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "baa14d9c3fbba59836ffbaac7f736a2f48d4a5fc209a2103d1e1b898915232a5" +checksum = "f8e9d37215f52e29b5ad9d9065c476168ffb888cd2ad93b0ae153f22304851a0" dependencies = [ "capacity_builder", - "deno_error 0.6.1", + "chrono", + "deno_error", "deno_path_util", "deno_terminal", "deno_unsync", + "dunce", "fqdn", - "ipnetwork", + "ipnet", "libc", "log", - "nix 0.27.1", + "nix 0.30.1", "once_cell", "parking_lot", - "percent-encoding", "serde", "serde_json", "sys_traits", - "temp_deno_which", "thiserror 2.0.21", + "unicode-normalization", "url", - "winapi", + "which 8.0.6", + "windows-sys 0.59.0", +] + +[[package]] +name = "deno_signals" +version = "0.50.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "038ed4b09a6d36c81ac436c4d10076e9e01cc67faf93d701c541887f8eeddf36" +dependencies = [ + "libc", + "signal-hook", + "thiserror 2.0.21", + "tokio", "windows-sys 0.59.0", ] [[package]] name = "deno_subprocess_windows" -version = "0.4.0" +version = "0.54.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7bc6b10059f0ccb14c6e0319c5275f0407fb2f9ffe405cd555700561999ea4bf" +checksum = "04d4f451b2f94d2847de4895d39ca5aa264b90af34e72fd79ddc1d0da05a5ea0" dependencies = [ - "fastrand 2.5.0", "futures-channel", "libc", + "rand 0.8.5", "windows-sys 0.59.0", ] [[package]] name = "deno_telemetry" -version = "0.31.0" +version = "0.81.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "377581966bd34e85ce230f7f4b1dee7e16c6bf7f845b4690648507ce7ba105d9" +checksum = "8831dc236ed1c66a90c8dfddab93ad44fbb28950fc476c78306b9100233ff694" dependencies = [ "async-trait", "deno_core", - "deno_error 0.6.1", + "deno_error", + "deno_net", + "deno_signals", + "deno_terminal", "deno_tls", + "flate2", "http-body-util", - "hyper 1.11.1", + "hyper 1.12.0", "hyper-rustls 0.27.10", "hyper-util", "log", "once_cell", - "opentelemetry 0.27.1", - "opentelemetry-http 0.27.0", - "opentelemetry-otlp 0.27.0", - "opentelemetry-semantic-conventions 0.27.0", - "opentelemetry_sdk 0.27.1", + "opentelemetry 0.32.0", + "opentelemetry-http 0.32.0", + "opentelemetry-otlp 0.32.0", + "opentelemetry-proto 0.32.0", + "opentelemetry-semantic-conventions 0.32.1", + "opentelemetry_sdk 0.32.1", "pin-project", + "prost 0.14.4", "serde", + "sys_traits", "thiserror 2.0.21", "tokio", + "tokio-vsock", + "tower-service", ] [[package]] name = "deno_terminal" -version = "0.2.2" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23f71c27009e0141dedd315f1dfa3ebb0a6ca4acce7c080fac576ea415a465f6" +checksum = "f3ba8041ae7319b3ca6a64c399df4112badcbbe0868b4517637647614bede4be" dependencies = [ "once_cell", "termcolor", @@ -4231,23 +4410,40 @@ dependencies = [ [[package]] name = "deno_tls" -version = "0.196.0" +version = "0.246.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7835eb6a8d114703b0293573fbb8885bc2c81e8c35363b9e89391dd3c541fa6" +checksum = "6229f7a00441b2acbf7fae58dd14bded0e835f212653402083430826e640f99d" dependencies = [ "deno_core", - "deno_error 0.6.1", + "deno_error", "deno_native_certs", - "rustls 0.23.35", + "log", + "rustls 0.23.40", "rustls-pemfile 2.2.0", "rustls-tokio-stream", - "rustls-webpki 0.102.8", + "rustls-webpki 0.103.15", "serde", "thiserror 2.0.21", "tokio", "webpki-roots 0.26.11", ] +[[package]] +name = "deno_tunnel" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e432929e0c167f2a003b963ee406c714fcd947a87d4f9edc7b3255e87be9881d" +dependencies = [ + "pin-project", + "quinn", + "serde", + "serde_json", + "socket2 0.6.5", + "thiserror 2.0.21", + "tokio", + "tracing", +] + [[package]] name = "deno_unsync" version = "0.4.4" @@ -4260,42 +4456,43 @@ dependencies = [ ] [[package]] -name = "deno_url" -version = "0.209.0" +name = "deno_v8" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8cccbb10fb59f29b04161055a65f7fe83364f9e8317d4d2ce3bd7979faf63a87" +checksum = "85cdff08dedf79da6a40f97554f47d59d419ba3df8061b8b31789cd2cbe00433" dependencies = [ - "deno_core", - "deno_error 0.6.1", - "urlpattern", + "v8", + "v8x", ] [[package]] name = "deno_web" -version = "0.240.0" +version = "0.290.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d6b8a97cc90b6aaea20fe200cfa7e5b6953dd33ecda0af50d6d360387c33df4" +checksum = "7510967ba4ec695959c1a518748c3b564fd9ca7a8f3eb13bd397707a3ca232da" dependencies = [ "async-trait", - "base64-simd", + "brotli 6.0.0", "bytes", + "cssparser", "deno_core", - "deno_error 0.6.1", - "deno_permissions", + "deno_error", "encoding_rs", "flate2", "futures", "serde", "thiserror 2.0.21", "tokio", + "unicode-normalization", + "urlpattern", "uuid", ] [[package]] name = "deno_webidl" -version = "0.209.0" +version = "0.259.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a20bbfac0cf15918f7cbd0b55c366ac20ad65c825f362fa684c3104984254d3b" +checksum = "40af42313723a8d3114c4e5b007abd1cab4400a0b68929794c556b2e792409c7" dependencies = [ "deno_core", ] @@ -4306,13 +4503,37 @@ version = "0.7.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e7c1832837b905bbfb5101e07cc24c8deddf52f93225eee6ead5f4d63d53ddcb" dependencies = [ - "const-oid", + "const-oid 0.9.6", "der_derive", "flagset", "pem-rfc7468", "zeroize", ] +[[package]] +name = "der" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a878c850e9e421b20262e9b41f9c860e4785fa07541c266b62ff9d1ef998a80a" +dependencies = [ + "const-oid 0.10.2", + "zeroize", +] + +[[package]] +name = "der-parser" +version = "8.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbd676fbbab537128ef0278adb5576cf363cff6aa22a7b24effe97347cfab61e" +dependencies = [ + "asn1-rs 0.5.2", + "displaydoc", + "nom", + "num-bigint", + "num-traits", + "rusticata-macros", +] + [[package]] name = "der-parser" version = "9.0.0" @@ -4361,6 +4582,25 @@ dependencies = [ "serde_core", ] +[[package]] +name = "derive-io" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "58cc7f4740088458993d183a200fe56e62378736f94bf0e2dd45807407e7bb94" +dependencies = [ + "derive-io-macros", + "tokio", +] + +[[package]] +name = "derive-io-macros" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d0a1bd7eeab72097740967d03d53db5fbaf8e3c0dd471ebdefa43ce445a20a6" +dependencies = [ + "proc-macro2", +] + [[package]] name = "derive_builder" version = "0.12.0" @@ -4450,7 +4690,6 @@ dependencies = [ "proc-macro2", "quote", "syn 2.0.119", - "unicode-xid", ] [[package]] @@ -4461,7 +4700,7 @@ checksum = "799a97264921d8623a957f6c3b9011f3b5492f557bbb7a5a19b7fa6d06ba8dcb" dependencies = [ "proc-macro2", "quote", - "rustc_version 0.4.1", + "rustc_version", "syn 2.0.119", ] @@ -4501,7 +4740,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ "block-buffer 0.10.4", - "const-oid", + "const-oid 0.9.6", "crypto-common 0.1.7", "subtle", ] @@ -4513,10 +4752,43 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" dependencies = [ "block-buffer 0.12.1", + "const-oid 0.10.2", "crypto-common 0.2.2", "ctutils", ] +[[package]] +name = "diplomat" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8b37a6cea72eb6df5143d3e091d4c34cb0cf4046dd1ddcc082988886fe1f809" +dependencies = [ + "diplomat_core", + "proc-macro2", + "quote", + "syn 3.0.6", +] + +[[package]] +name = "diplomat-runtime" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae517853d9f82b2d144393b1c32c34c46c2e361de2b6c44a8fdfce23beb211cd" + +[[package]] +name = "diplomat_core" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73b18e806f9ecb92df828d080efe47984ae1193958295d4a84fb8b32c9a1aa1b" +dependencies = [ + "proc-macro2", + "quote", + "serde", + "smallvec", + "strck", + "syn 3.0.6", +] + [[package]] name = "dirs" version = "4.0.0" @@ -4674,6 +4946,27 @@ dependencies = [ "text_lines", ] +[[package]] +name = "drop_bomb" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9bda8e21c04aca2ae33ffc2fd8c23134f3cac46db123ba97bd9d3f3b8a4a85e1" + +[[package]] +name = "dtoa" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c3cf4824e2d5f025c7b531afcb2325364084a16806f6d47fbc1f5fbd9960590" + +[[package]] +name = "dtoa-short" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd1511a7b6a56299bd043a9c167a6d2bfb37bf84a6dfceaba651168adfb43c87" +dependencies = [ + "dtoa", +] + [[package]] name = "duct" version = "0.13.7" @@ -4720,12 +5013,12 @@ version = "0.16.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ee27f32b5c5292967d2d4a9d7f1e0b0aed2c15daded5a60300e4abb9d8020bca" dependencies = [ - "der", + "der 0.7.10", "digest 0.10.7", - "elliptic-curve", + "elliptic-curve 0.13.8", "rfc6979", - "signature", - "spki", + "signature 2.2.0", + "spki 0.7.3", ] [[package]] @@ -4734,8 +5027,8 @@ version = "2.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "115531babc129696a58c64a4fef0a8bf9e9698629fb97e9e40767d235cfbcd53" dependencies = [ - "pkcs8", - "signature", + "pkcs8 0.10.2", + "signature 2.2.0", ] [[package]] @@ -4749,21 +5042,35 @@ dependencies = [ "rand_core 0.6.4", "serde", "sha2 0.10.9", - "signature", + "signature 2.2.0", "subtle", "zeroize", ] [[package]] -name = "ed448-goldilocks" -version = "0.8.3" +name = "ed448" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06924531e9e90130842b012e447f85bdaf9161bc8a0f8092be8cb70b01ebe092" +checksum = "ae112a25f86ae3598d4e8533ed1e65149cec6eb21918e7a6f4c06dddec370263" dependencies = [ - "fiat-crypto 0.1.20", - "hex", + "pkcs8 0.11.0", + "signature 3.0.0", +] + +[[package]] +name = "ed448-goldilocks" +version = "0.14.0-pre.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b805154de2e68f59874ec217ca36790dcffe500cd872c60fe509d28d0814a74d" +dependencies = [ + "ed448", + "elliptic-curve 0.14.1", + "hash2curve", + "rand_core 0.10.1", + "serdect 0.4.3", + "shake", + "signature 3.0.0", "subtle", - "zeroize", ] [[package]] @@ -4780,9 +5087,9 @@ dependencies = [ [[package]] name = "either" -version = "1.18.0" +version = "1.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "252afb9ae5eaa683babdc6a068b3f5726eb19e05070c731f9b2a23a7c3e8ed34" +checksum = "0e9c71c2167ca323c882b99918929403426e2373ea17242ff5653e0d5e1058be" dependencies = [ "serde", ] @@ -4793,20 +5100,39 @@ version = "0.13.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b5e6043086bf7973472e0c7dff2142ea0b680d30e18d9cc40f267efbf222bd47" dependencies = [ - "base16ct", + "base16ct 0.2.0", "base64ct", - "crypto-bigint", + "crypto-bigint 0.5.5", "digest 0.10.7", - "ff", + "ff 0.13.1", "generic-array", - "group", + "group 0.13.0", "hkdf", "pem-rfc7468", - "pkcs8", + "pkcs8 0.10.2", "rand_core 0.6.4", - "sec1", + "sec1 0.7.3", "serde_json", - "serdect", + "serdect 0.2.0", + "subtle", + "zeroize", +] + +[[package]] +name = "elliptic-curve" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d65aa39b3a5c1c9c1b745c9a019234bb7a21b77abcb4f4d266d706e2d577d65" +dependencies = [ + "base16ct 1.0.0", + "crypto-bigint 0.7.5", + "crypto-common 0.2.2", + "ff 0.14.0", + "group 0.14.0", + "hybrid-array", + "pkcs8 0.11.0", + "rand_core 0.10.1", + "sec1 0.8.1", "subtle", "zeroize", ] @@ -5005,6 +5331,18 @@ dependencies = [ "regex-syntax 0.8.11", ] +[[package]] +name = "fastbloom" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef975e30683b2d965054bb0a836f8973857c4ebf6acf274fe46617cd285060d8" +dependencies = [ + "foldhash 0.2.0", + "libm", + "portable-atomic", + "siphasher 1.0.4", +] + [[package]] name = "fastdivide" version = "0.4.2" @@ -5037,10 +5375,14 @@ dependencies = [ ] [[package]] -name = "fiat-crypto" -version = "0.1.20" +name = "ff" +version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e825f6987101665dea6ec934c09ec6d721de7bc1bf92248e1d5810c8cd636b77" +checksum = "a1f686ab92a9fb0eaf188f6c6c87b89490baa6fdb0db4544ba4dc47f7942489f" +dependencies = [ + "rand_core 0.10.1", + "subtle", +] [[package]] name = "fiat-crypto" @@ -5064,6 +5406,30 @@ version = "0.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "aedcfb3409746eddb02b9e19ebda1c3394f759a152e48ee875a0844d1b955484" +[[package]] +name = "fips203" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8bdb6454f692ca2a2b45cd554c6828c639d7f9c968cf83a678899ec4443a280" +dependencies = [ + "rand_core 0.6.4", + "sha3", + "subtle", + "zeroize", +] + +[[package]] +name = "fips205" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f5626bf5534df4ebdbd2536465d7eaa8a9dc2cdeb7e036e0ecf291dcc80ffb6" +dependencies = [ + "rand_core 0.6.4", + "sha2 0.10.9", + "sha3", + "zeroize", +] + [[package]] name = "fixedbitset" version = "0.5.7" @@ -5083,7 +5449,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "35f6839d7b3b98adde531effaf34f0c2badc6f4735d26fe74709d8e513a96ef3" dependencies = [ "bitflags 2.13.2", - "rustc_version 0.4.1", + "rustc_version", ] [[package]] @@ -5166,9 +5532,9 @@ dependencies = [ [[package]] name = "fqdn" -version = "0.3.12" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb540cf7bc4fe6df9d8f7f0c974cfd0dce8ed4e9e8884e73433b503ee78b4e7d" +checksum = "886ac788f62d16d6b0f26b2fa762b34ef16ebfb4b624c2c15fbcadc9173c0f72" [[package]] name = "from_variant" @@ -5186,17 +5552,6 @@ version = "1.20260821.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "28c70238056ad1dbfdf035222a571d2e9baf5b6a4c308cfa87bc8cbe7f2f3840" -[[package]] -name = "fs3" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb17cf6ed704f72485332f6ab65257460c4f9f3083934cf402bf9f5b3b600a90" -dependencies = [ - "libc", - "rustc_version 0.2.3", - "winapi", -] - [[package]] name = "fs4" version = "0.13.1" @@ -5526,6 +5881,31 @@ dependencies = [ "zeroize", ] +[[package]] +name = "get-size-derive2" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28f0daa179069133df60523e0f59d0a6bd2d3db9ccae01b630896d269c4a802e" +dependencies = [ + "attribute-derive", + "quote", + "syn 3.0.6", +] + +[[package]] +name = "get-size2" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "050a5d07359ebfac7f0b74916d94cfa30575e45fdc7e9bfbb0f76d5d4203325a" +dependencies = [ + "compact_str 0.10.0", + "get-size-derive2", + "hashbrown 0.17.1", + "ordermap", + "smallvec", + "thin-vec", +] + [[package]] name = "gethostname" version = "0.4.3" @@ -5607,12 +5987,6 @@ dependencies = [ "polyval", ] -[[package]] -name = "gimli" -version = "0.32.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e629b9b98ef3dd8afe6ca2bd0f89306cec16d43d907889945bc5d6687f2f13c7" - [[package]] name = "git-version" version = "0.3.9" @@ -5675,7 +6049,7 @@ dependencies = [ "google-cloud-metadata", "google-cloud-token", "hex", - "hmac", + "hmac 0.12.1", "home", "jsonwebtoken 9.3.1", "path-clean", @@ -5714,7 +6088,7 @@ version = "0.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "886aa8ec755382a1fdf4651f6e6ec01f2f3bf49f2cb0f068b9a74cafd574a715" dependencies = [ - "prost", + "prost 0.13.5", "prost-types", "tonic 0.12.3", ] @@ -5776,11 +6150,22 @@ version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0f9ef7462f7c099f518d754361858f86d8a07af53ba9af0fe635bbccb151a63" dependencies = [ - "ff", + "ff 0.13.1", "rand_core 0.6.4", "subtle", ] +[[package]] +name = "group" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fd1a1c7a5206c5b7a3f5a0d7ccd3ff85d0c8f5133d62a02680255b0004af5f4" +dependencies = [ + "ff 0.14.0", + "rand_core 0.10.1", + "subtle", +] + [[package]] name = "gzip-header" version = "1.1.0" @@ -5811,9 +6196,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.4.19" +version = "0.4.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef8e5e5a340588f4452631496976cf8636d4a7ecf600239fdc27615d2530bc16" +checksum = "7d29020232d6aa3fb1daca64c1127cf662cf97f254ae16c18c05b8ab635fc118" dependencies = [ "atomic-waker", "bytes", @@ -5843,6 +6228,16 @@ dependencies = [ "zerocopy", ] +[[package]] +name = "hash2curve" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1eaf40612d7d854743e7189228a6d528f0f6e8502cf6a0cb831d28a218b7f3f6" +dependencies = [ + "digest 0.11.3", + "elliptic-curve 0.14.1", +] + [[package]] name = "hashbrown" version = "0.12.3" @@ -5990,11 +6385,10 @@ dependencies = [ [[package]] name = "hickory-proto" -version = "0.25.0-alpha.5" +version = "0.25.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d00147af6310f4392a31680db52a3ed45a2e0f68eb18e8c3fe5537ecc96d9e2" +checksum = "f8a6fe56c0038198998a6f217ca4e7ef3a5e51f46163bd6dd60b5c71ca6c6502" dependencies = [ - "async-recursion", "async-trait", "cfg-if", "data-encoding", @@ -6006,6 +6400,7 @@ dependencies = [ "ipnet", "once_cell", "rand 0.9.5", + "ring 0.17.14", "serde", "thiserror 2.0.21", "tinyvec", @@ -6016,9 +6411,9 @@ dependencies = [ [[package]] name = "hickory-resolver" -version = "0.25.0-alpha.5" +version = "0.25.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5762f69ebdbd4ddb2e975cd24690bf21fe6b2604039189c26acddbc427f12887" +checksum = "dc62a9a99b0bfb44d2ab95a7208ac952d31060efc16241c87eaf36406fecf87a" dependencies = [ "cfg-if", "futures-util", @@ -6042,7 +6437,7 @@ version = "0.12.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7b5f8eb2ad728638ea2c7d47a21db23b7b58a72ed6a38256b8a1849f15fbbdf7" dependencies = [ - "hmac", + "hmac 0.12.1", ] [[package]] @@ -6054,6 +6449,15 @@ dependencies = [ "digest 0.10.7", ] +[[package]] +name = "hmac" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6303bc9732ae41b04cb554b844a762b4115a61bfaa81e3e83050991eeb56863f" +dependencies = [ + "digest 0.11.3", +] + [[package]] name = "home" version = "0.5.12" @@ -6171,7 +6575,7 @@ dependencies = [ "futures", "http 1.5.0", "http-body-util", - "hyper 1.11.1", + "hyper 1.12.0", "hyper-rustls 0.26.0", "hyper-tls", "hyper-tungstenite", @@ -6201,7 +6605,9 @@ version = "0.4.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "27f864f10dfb56725ce5ce5472bc52252c8f93a4ab86327122cebf62c5f59a17" dependencies = [ + "subtle", "typenum", + "zeroize", ] [[package]] @@ -6230,15 +6636,15 @@ dependencies = [ [[package]] name = "hyper" -version = "1.11.1" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27b501faa50e7a26c3d3560ca625132f4078a17771f4810baf70475ae48cbe43" +checksum = "2c3e324da4c95177d6291d4c8730197c0d1822f8a9766814a4a44fa5ab797c9c" dependencies = [ "atomic-waker", "bytes", "futures-channel", "futures-core", - "h2 0.4.19", + "h2 0.4.20", "http 1.5.0", "http-body 1.1.0", "httparse", @@ -6260,7 +6666,7 @@ dependencies = [ "futures-util", "headers", "http 1.5.0", - "hyper 1.11.1", + "hyper 1.12.0", "hyper-rustls 0.27.10", "hyper-util", "pin-project-lite", @@ -6276,7 +6682,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fab3637d6b04a8037af8a266fdf6cf92ea957e8c53981a2bf6136572531025bf" dependencies = [ "hex", - "hyper 1.11.1", + "hyper 1.12.0", "hyper-util", "pin-project-lite", "tokio", @@ -6307,7 +6713,7 @@ checksum = "a0bea761b46ae2b24eb4aef630d8d1c398157b6fc29e6350ecf090a0b70c952c" dependencies = [ "futures-util", "http 1.5.0", - "hyper 1.11.1", + "hyper 1.12.0", "hyper-util", "log", "rustls 0.22.4", @@ -6325,10 +6731,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dfa8e654703247911e29c23fbeaa261834bd9bb74efba2f9acddc37bfb127f53" dependencies = [ "http 1.5.0", - "hyper 1.11.1", + "hyper 1.12.0", "hyper-util", "log", - "rustls 0.23.35", + "rustls 0.23.40", "rustls-native-certs 0.8.4", "tokio", "tokio-rustls 0.26.6", @@ -6342,7 +6748,7 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2b90d566bffbce6a75bd8b09a05aa8c2cb1fabb6cb348f8840c9e4c90a0d83b0" dependencies = [ - "hyper 1.11.1", + "hyper 1.12.0", "hyper-util", "pin-project-lite", "tokio", @@ -6357,7 +6763,7 @@ checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" dependencies = [ "bytes", "http-body-util", - "hyper 1.11.1", + "hyper 1.12.0", "hyper-util", "native-tls", "tokio", @@ -6372,7 +6778,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7a343d17fe7885302ed7252767dc7bb83609a874b6ff581142241ec4b73957ad" dependencies = [ "http-body-util", - "hyper 1.11.1", + "hyper 1.12.0", "hyper-util", "pin-project-lite", "tokio", @@ -6393,7 +6799,7 @@ dependencies = [ "http 1.5.0", "http-body 1.1.0", "httparse", - "hyper 1.11.1", + "hyper 1.12.0", "ipnet", "libc", "percent-encoding", @@ -6414,7 +6820,7 @@ checksum = "986c5ce3b994526b3cd75578e62554abd09f0899d6206de48b3e96ab34ccc8c7" dependencies = [ "hex", "http-body-util", - "hyper 1.11.1", + "hyper 1.12.0", "hyper-util", "pin-project-lite", "tokio", @@ -6445,6 +6851,28 @@ dependencies = [ "cc", ] +[[package]] +name = "icu_calendar" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "58655df2f728e46e4eee80bddebefacf52ec3e77cdb925014b47c5a5905eb55c" +dependencies = [ + "calendrical_calculations", + "displaydoc", + "icu_calendar_data", + "icu_locale_core", + "icu_locale_fallback", + "icu_provider", + "tinystr", + "zerovec", +] + +[[package]] +name = "icu_calendar_data" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc00caaa3fb3201ff7a18aa458e8c3ab042b9da154cfdf948bdde3936c31c36e" + [[package]] name = "icu_collections" version = "2.3.0" @@ -6668,10 +7096,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8bb03732005da905c88227371639bf1ad885cc712789c011c31c5fb3ab3ccf02" [[package]] -name = "inventory" -version = "0.3.24" +name = "interpolator" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4f0c30c76f2f4ccee3fe55a2435f691ca00c0e4bd87abe4f4a851b1d4dac39b" +checksum = "71dd52191aae121e8611f1e8dc3e324dd0dd1dee1e6dd91d10ee07a3cfb4d9d8" + +[[package]] +name = "inventory" +version = "0.3.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6928282826c822ad91bf1c9a1cb90a30ba1c26770749929b4656cd6be829cd7c" dependencies = [ "rustversion", ] @@ -6687,17 +7121,6 @@ dependencies = [ "windows-sys 0.48.0", ] -[[package]] -name = "io-uring" -version = "0.7.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed3bd0ecfbb87805f538bb7b32e5239ca0763890c623e349860ecba69469f2bb" -dependencies = [ - "bitflags 2.13.2", - "cfg-if", - "libc", -] - [[package]] name = "ipconfig" version = "0.3.4" @@ -6717,15 +7140,6 @@ version = "2.12.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "791930b43c0d5973160d90a8f3894509f2b273430f5c5c73b668636d0287c5c0" -[[package]] -name = "ipnetwork" -version = "0.20.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf466541e9d546596ee94f9f69590f89473455f88372423e0008fc1a7daf100e" -dependencies = [ - "serde", -] - [[package]] name = "is-macro" version = "0.3.8" @@ -6786,6 +7200,15 @@ dependencies = [ "either", ] +[[package]] +name = "itertools" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b4baf93f58d4425749ca49a51c50ebab072c5df6994d08fed93541c331481dc" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.18" @@ -6793,10 +7216,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" [[package]] -name = "jiff" -version = "0.2.37" +name = "ixdtf" +version = "0.6.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ab1baf72f08796de0260609515130699b890ac25f30e610ad894bc5856cafdb" +checksum = "4d3667095d64c3ecffc96463a21157b04bf3e252f6e8d5750b20c02e33c194e3" + +[[package]] +name = "jiff" +version = "0.2.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2b005715dcbeb0089a3c0dab99f2ff1cc3b2525323552703d648585d342a383" dependencies = [ "defmt", "jiff-core", @@ -6821,9 +7250,9 @@ dependencies = [ [[package]] name = "jiff-static" -version = "0.2.37" +version = "0.2.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "378268a1116ad67ae6228701118ac9f491d78fda38a40a1f1a9e1348de6f7212" +checksum = "2cc9817253cf7c7ee4684451bd327e88d6f3658014e54a29198625590650695c" dependencies = [ "jiff-core", "proc-macro2", @@ -6833,9 +7262,9 @@ dependencies = [ [[package]] name = "jiff-tzdb" -version = "0.1.8" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "142bd39932ad231f10513df9ab62661fead8719872150b7ad02a2df79f4e141e" +checksum = "fa8377070c6bae868759445e5a77f66d84f0b72f3a054bfb00e6d038b8282da7" [[package]] name = "jiff-tzdb-platform" @@ -6871,7 +7300,7 @@ checksum = "a00109accc170f0bdb141fed3e393c565b6f5e072365c3bd58f5b062591560a3" dependencies = [ "proc-macro2", "quote", - "rustc_version 0.4.1", + "rustc_version", "simd_cesu8", "syn 2.0.119", ] @@ -6979,7 +7408,7 @@ dependencies = [ "pem", "serde", "serde_json", - "signature", + "signature 2.2.0", "simple_asn1", "zeroize", ] @@ -7006,6 +7435,25 @@ dependencies = [ "serde_json", ] +[[package]] +name = "keccak" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb26cec98cce3a3d96cbb7bced3c4b16e3d13f27ec56dbd62cbc8f39cfb9d653" +dependencies = [ + "cpufeatures 0.2.17", +] + +[[package]] +name = "keccak" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8f198d1db720e4940b5a493201d199d9f24f568f8f746bd13706243a2f71598" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.1", +] + [[package]] name = "keyed_priority_queue" version = "0.4.2" @@ -7068,7 +7516,7 @@ dependencies = [ "http 1.5.0", "http-body 1.1.0", "http-body-util", - "hyper 1.11.1", + "hyper 1.12.0", "hyper-http-proxy", "hyper-rustls 0.27.10", "hyper-timeout", @@ -7077,7 +7525,7 @@ dependencies = [ "k8s-openapi", "kube-core", "pem", - "rustls 0.23.35", + "rustls 0.23.40", "secrecy", "serde", "serde_json", @@ -7150,12 +7598,6 @@ dependencies = [ "tracing", ] -[[package]] -name = "lalrpop-util" -version = "0.20.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "507460a910eb7b32ee961886ff48539633b788a36b65692b95f225b844c82553" - [[package]] name = "lapin" version = "2.5.5" @@ -7252,9 +7694,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.189" +version = "0.2.190" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" +checksum = "ce5d3ddc6d3fa000eb1536d85e147bfe31aacaba692ed6a876f95cb7c855be78" [[package]] name = "libffi-sys" @@ -7566,7 +8008,7 @@ dependencies = [ "base64 0.22.1", "gethostname", "mail-builder", - "rustls 0.23.35", + "rustls 0.23.40", "rustls-pki-types", "smtp-proto", "tokio", @@ -7574,76 +8016,30 @@ dependencies = [ "webpki-roots 0.26.11", ] -[[package]] -name = "malachite" -version = "0.4.22" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2fbdf9cb251732db30a7200ebb6ae5d22fe8e11397364416617d2c2cf0c51cb5" -dependencies = [ - "malachite-base", - "malachite-nz", - "malachite-q", -] - -[[package]] -name = "malachite-base" -version = "0.4.22" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ea0ed76adf7defc1a92240b5c36d5368cfe9251640dcce5bd2d0b7c1fd87aeb" -dependencies = [ - "hashbrown 0.14.5", - "itertools 0.11.0", - "libm", - "ryu", -] - -[[package]] -name = "malachite-bigint" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d149aaa2965d70381709d9df4c7ee1fc0de1c614a4efc2ee356f5e43d68749f8" -dependencies = [ - "derive_more 1.0.0", - "malachite", - "num-integer", - "num-traits", - "paste", -] - -[[package]] -name = "malachite-nz" -version = "0.4.22" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34a79feebb2bc9aa7762047c8e5495269a367da6b5a90a99882a0aeeac1841f7" -dependencies = [ - "itertools 0.11.0", - "libm", - "malachite-base", -] - -[[package]] -name = "malachite-q" -version = "0.4.22" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50f235d5747b1256b47620f5640c2a17a88c7569eebdf27cd9cb130e1a619191" -dependencies = [ - "itertools 0.11.0", - "malachite-base", - "malachite-nz", -] - [[package]] name = "manyhow" version = "0.11.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b33efb3ca6d3b07393750d4030418d594ab1139cee518f0dc88db70fec873587" dependencies = [ - "manyhow-macros", + "manyhow-macros 0.11.4", "proc-macro2", "quote", "syn 2.0.119", ] +[[package]] +name = "manyhow" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c69ea6e469ca785c846b25dd69eac9e9c916b51d8ada1f212866787ee1da12d" +dependencies = [ + "manyhow-macros 0.14.0", + "proc-macro2", + "quote", + "syn 3.0.6", +] + [[package]] name = "manyhow-macros" version = "0.11.4" @@ -7655,6 +8051,17 @@ dependencies = [ "quote", ] +[[package]] +name = "manyhow-macros" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22fc5c1b2c4000056dae3619fea36e79ce79dabfee4907d2dcad6612b189e370" +dependencies = [ + "proc-macro-utils", + "proc-macro2", + "quote", +] + [[package]] name = "mappable-rc" version = "0.1.1" @@ -7709,6 +8116,16 @@ dependencies = [ "digest 0.10.7", ] +[[package]] +name = "md-5" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69b6441f590336821bb897fb28fc622898ccceb1d6cea3fde5ea86b090c4de98" +dependencies = [ + "cfg-if", + "digest 0.11.3", +] + [[package]] name = "md5" version = "0.6.1" @@ -7830,9 +8247,9 @@ dependencies = [ [[package]] name = "mio" -version = "1.2.3" +version = "1.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b18443e9c262bfe8fa82f51666e2642c53393f7e5c27b3e1aeab922cff5b9d8" +checksum = "1788edb87fdc09c7e26304471e2f5be8cdefb1b6930d6e3985fc02ff53bf86ee" dependencies = [ "libc", "wasi 0.11.1+wasi-snapshot-preview1", @@ -7912,7 +8329,7 @@ checksum = "a4db8a44120571277accfaa3f3d91e7d3989d601d817c2fc01a9391b86135666" dependencies = [ "darling 0.23.0", "heck", - "manyhow", + "manyhow 0.11.4", "num-bigint", "proc-macro-crate", "proc-macro2", @@ -8046,6 +8463,7 @@ dependencies = [ "cfg-if", "cfg_aliases", "libc", + "memoffset", ] [[package]] @@ -8265,7 +8683,6 @@ checksum = "c89e69e7e0f03bea5ef08013795c25018e101932225a656383bd384495ecc367" dependencies = [ "num-integer", "num-traits", - "rand 0.8.5", "serde", ] @@ -8409,15 +8826,6 @@ dependencies = [ "url", ] -[[package]] -name = "object" -version = "0.37.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff76201f031d8863c38aa7f905eca4f53abbfa15f609db4277d44cd8938f33fe" -dependencies = [ - "memchr", -] - [[package]] name = "object" version = "0.39.1" @@ -8442,7 +8850,7 @@ dependencies = [ "http-body-util", "httparse", "humantime", - "hyper 1.11.1", + "hyper 1.12.0", "itertools 0.14.0", "md-5 0.10.6", "parking_lot", @@ -8464,6 +8872,18 @@ dependencies = [ "web-time", ] +[[package]] +name = "ocb3" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c196e0276c471c843dd5777e7543a36a298a4be942a2a688d8111cd43390dedb" +dependencies = [ + "aead", + "cipher 0.4.4", + "ctr", + "subtle", +] + [[package]] name = "odpic-sys" version = "0.1.1" @@ -8473,6 +8893,15 @@ dependencies = [ "cc", ] +[[package]] +name = "oid-registry" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9bedf36ffb6ba96c2eb7144ef6270557b52e54b20c0a8e1eb2ff99a6c6959bff" +dependencies = [ + "asn1-rs 0.5.2", +] + [[package]] name = "oid-registry" version = "0.7.1" @@ -8502,6 +8931,10 @@ name = "once_cell" version = "1.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" +dependencies = [ + "critical-section", + "portable-atomic", +] [[package]] name = "once_cell_polyfill" @@ -8559,7 +8992,7 @@ dependencies = [ "chrono", "dyn-clone", "ed25519-dalek", - "hmac", + "hmac 0.12.1", "http 1.5.0", "itertools 0.10.5", "log", @@ -8641,23 +9074,23 @@ dependencies = [ [[package]] name = "opentelemetry" -version = "0.27.1" +version = "0.30.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab70038c28ed37b97d8ed414b6429d343a8bbf44c9f79ec854f3a643029ba6d7" +checksum = "aaf416e4cb72756655126f7dd7bb0af49c674f4c1b9903e80c009e0c37e552e6" dependencies = [ "futures-core", "futures-sink", "js-sys", "pin-project-lite", - "thiserror 1.0.69", + "thiserror 2.0.21", "tracing", ] [[package]] name = "opentelemetry" -version = "0.30.0" +version = "0.32.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aaf416e4cb72756655126f7dd7bb0af49c674f4c1b9903e80c009e0c37e552e6" +checksum = "b0142c63252a9e054e68a4c61a5778f7b14f576274d593f8ce883d191a099682" dependencies = [ "futures-core", "futures-sink", @@ -8679,18 +9112,6 @@ dependencies = [ "tracing-subscriber", ] -[[package]] -name = "opentelemetry-http" -version = "0.27.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10a8a7f5f6ba7c1b286c2fbca0454eaba116f63bbe69ed250b642d36fbb04d80" -dependencies = [ - "async-trait", - "bytes", - "http 1.5.0", - "opentelemetry 0.27.1", -] - [[package]] name = "opentelemetry-http" version = "0.30.0" @@ -8705,24 +9126,15 @@ dependencies = [ ] [[package]] -name = "opentelemetry-otlp" -version = "0.27.0" +name = "opentelemetry-http" +version = "0.32.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91cf61a1868dacc576bf2b2a1c3e9ab150af7272909e80085c3173384fe11f76" +checksum = "5683015d09e2df236ef005b17f6f196f0d5f6313c4fa43a7b6a53b52776e4331" dependencies = [ "async-trait", - "futures-core", + "bytes", "http 1.5.0", - "opentelemetry 0.27.1", - "opentelemetry-http 0.27.0", - "opentelemetry-proto 0.27.0", - "opentelemetry_sdk 0.27.1", - "prost", - "serde_json", - "thiserror 1.0.69", - "tokio", - "tonic 0.12.3", - "tracing", + "opentelemetry 0.32.0", ] [[package]] @@ -8736,7 +9148,7 @@ dependencies = [ "opentelemetry-http 0.30.0", "opentelemetry-proto 0.30.0", "opentelemetry_sdk 0.30.0", - "prost", + "prost 0.13.5", "reqwest 0.12.28", "thiserror 2.0.21", "tokio", @@ -8745,17 +9157,19 @@ dependencies = [ ] [[package]] -name = "opentelemetry-proto" -version = "0.27.0" +name = "opentelemetry-otlp" +version = "0.32.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6e05acbfada5ec79023c85368af14abd0b307c015e9064d249b2a950ef459a6" +checksum = "9966929966d17620d7c316c643ba62631826e10021409357772d5eea84f62c35" dependencies = [ - "hex", - "opentelemetry 0.27.1", - "opentelemetry_sdk 0.27.1", - "prost", - "serde", - "tonic 0.12.3", + "http 1.5.0", + "opentelemetry 0.32.0", + "opentelemetry-http 0.32.0", + "opentelemetry-proto 0.32.0", + "opentelemetry_sdk 0.32.1", + "prost 0.14.4", + "serde_json", + "thiserror 2.0.21", ] [[package]] @@ -8768,16 +9182,24 @@ dependencies = [ "hex", "opentelemetry 0.30.0", "opentelemetry_sdk 0.30.0", - "prost", + "prost 0.13.5", "serde", "tonic 0.13.1", ] [[package]] -name = "opentelemetry-semantic-conventions" -version = "0.27.0" +name = "opentelemetry-proto" +version = "0.32.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc1b6902ff63b32ef6c489e8048c5e253e2e4a803ea3ea7e783914536eb15c52" +checksum = "56d658ba1faf63f7b9c492cfbe6e0ec365440a16132d3270c1065f7b33f1b638" +dependencies = [ + "base64 0.22.1", + "const-hex", + "opentelemetry 0.32.0", + "opentelemetry_sdk 0.32.1", + "prost 0.14.4", + "serde", +] [[package]] name = "opentelemetry-semantic-conventions" @@ -8786,25 +9208,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "83d059a296a47436748557a353c5e6c5705b9470ef6c95cfc52c21a8814ddac2" [[package]] -name = "opentelemetry_sdk" -version = "0.27.1" +name = "opentelemetry-semantic-conventions" +version = "0.32.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "231e9d6ceef9b0b2546ddf52335785ce41252bc7474ee8ba05bfad277be13ab8" -dependencies = [ - "async-trait", - "futures-channel", - "futures-executor", - "futures-util", - "glob", - "opentelemetry 0.27.1", - "percent-encoding", - "rand 0.8.5", - "serde_json", - "thiserror 1.0.69", - "tokio", - "tokio-stream", - "tracing", -] +checksum = "c913ac17a6c451661ee255f4625d143e51647ae78ebd969b75e41c4442f4fe47" [[package]] name = "opentelemetry_sdk" @@ -8824,6 +9231,24 @@ dependencies = [ "tokio-stream", ] +[[package]] +name = "opentelemetry_sdk" +version = "0.32.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b59f80e1ac4d5ff7a2db8fb6c80badb7f0f3f858211fba08dd9aaec750894f9" +dependencies = [ + "futures-channel", + "futures-executor", + "futures-util", + "opentelemetry 0.32.0", + "percent-encoding", + "portable-atomic", + "rand 0.9.5", + "thiserror 2.0.21", + "tokio", + "tokio-stream", +] + [[package]] name = "option-ext" version = "0.2.0" @@ -8875,6 +9300,15 @@ dependencies = [ "num-traits", ] +[[package]] +name = "ordermap" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16a756ddcdeb6c0efc8927bdf0ed112885815a35ba350472a467be5104f01ca1" +dependencies = [ + "indexmap 2.14.2", +] + [[package]] name = "os_pipe" version = "1.2.1" @@ -8913,10 +9347,10 @@ checksum = "3cae83056e7cb770211494a0ecf66d9fa7eba7d00977e5bb91f0e925b40b937f" dependencies = [ "cbc", "cms", - "der", + "der 0.7.10", "des 0.8.1", "hex", - "hmac", + "hmac 0.12.1", "pkcs12", "pkcs5", "rand 0.9.5", @@ -8934,7 +9368,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c9863ad85fa8f4460f9c48cb909d38a0d689dba1f6f6988a5e3e0d31071bcd4b" dependencies = [ "ecdsa", - "elliptic-curve", + "elliptic-curve 0.13.8", "primeorder", "sha2 0.10.9", ] @@ -8946,7 +9380,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fe42f1670a52a47d448f14b6a5c61dd78fce51856e68edaa38f7ae3a46b8d6b6" dependencies = [ "ecdsa", - "elliptic-curve", + "elliptic-curve 0.13.8", "primeorder", "sha2 0.10.9", ] @@ -8957,9 +9391,9 @@ version = "0.13.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0fc9e2161f1f215afdfce23677034ae137bbd45016a880c2eb3ba8eb95f085b2" dependencies = [ - "base16ct", + "base16ct 0.2.0", "ecdsa", - "elliptic-curve", + "elliptic-curve 0.13.8", "primeorder", "rand_core 0.6.4", "sha2 0.10.9", @@ -9080,7 +9514,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8ed6a7761f76e3b9f92dfb0a60a6a6477c61024b775147ff0973a02653abaf2" dependencies = [ "digest 0.10.7", - "hmac", + "hmac 0.12.1", ] [[package]] @@ -9198,7 +9632,7 @@ version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1fd6780a80ae0c52cc120a26a1a42c1ae51b247a253e4e06113d23d2c2edd078" dependencies = [ - "phf_macros", + "phf_macros 0.11.3", "phf_shared 0.11.3", ] @@ -9217,6 +9651,7 @@ version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c1562dc717473dbaa4c1f85a36410e03c047b2e7df7f45ee938fbef64ae7fadf" dependencies = [ + "phf_macros 0.13.1", "phf_shared 0.13.1", "serde", ] @@ -9227,7 +9662,7 @@ version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "aef8048c789fa5e851558d709946d6d79a8ff88c0440c587967f8e94bfb1216a" dependencies = [ - "phf_generator", + "phf_generator 0.11.3", "phf_shared 0.11.3", ] @@ -9241,19 +9676,42 @@ dependencies = [ "rand 0.8.5", ] +[[package]] +name = "phf_generator" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "135ace3a761e564ec88c03a77317a7c6b80bb7f7135ef2544dbe054243b89737" +dependencies = [ + "fastrand 2.5.0", + "phf_shared 0.13.1", +] + [[package]] name = "phf_macros" version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f84ac04429c13a7ff43785d75ad27569f2951ce0ffd30a3321230db2fc727216" dependencies = [ - "phf_generator", + "phf_generator 0.11.3", "phf_shared 0.11.3", "proc-macro2", "quote", "syn 2.0.119", ] +[[package]] +name = "phf_macros" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "812f032b54b1e759ccd5f8b6677695d5268c588701effba24601f6932f8269ef" +dependencies = [ + "phf_generator 0.13.1", + "phf_shared 0.13.1", + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "phf_shared" version = "0.11.3" @@ -9354,9 +9812,9 @@ version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c8ffb9f10fa047879315e6625af03c164b16962a5368d724ed16323b68ace47f" dependencies = [ - "der", - "pkcs8", - "spki", + "der 0.7.10", + "pkcs8 0.10.2", + "spki 0.7.3", ] [[package]] @@ -9366,10 +9824,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "695b3df3d3cc1015f12d70235e35b6b79befc5fa7a9b95b951eab1dd07c9efc2" dependencies = [ "cms", - "const-oid", - "der", + "const-oid 0.9.6", + "der 0.7.10", "digest 0.10.7", - "spki", + "spki 0.7.3", "x509-cert", "zeroize", ] @@ -9382,11 +9840,11 @@ checksum = "e847e2c91a18bfa887dd028ec33f2fe6f25db77db3619024764914affe8b69a6" dependencies = [ "aes 0.8.3", "cbc", - "der", + "der 0.7.10", "pbkdf2", "scrypt", "sha2 0.10.9", - "spki", + "spki 0.7.3", ] [[package]] @@ -9395,8 +9853,18 @@ version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" dependencies = [ - "der", - "spki", + "der 0.7.10", + "spki 0.7.3", +] + +[[package]] +name = "pkcs8" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "451913da69c775a56034ea8d9003d27ee8948e12443eae7c038ba100a4f21cb7" +dependencies = [ + "der 0.8.2", + "spki 0.8.1", ] [[package]] @@ -9471,7 +9939,7 @@ dependencies = [ [[package]] name = "postgres-native-tls" version = "0.5.2" -source = "git+https://github.com/MaterializeInc/rust-postgres?rev=78c1222577bb091d69bc22b1bc7ad01c14675abe#78c1222577bb091d69bc22b1bc7ad01c14675abe" +source = "git+https://github.com/windmill-labs/rust-postgres?rev=b3a6443aa1ae90a4f6c4b742103ba80584598f2e#b3a6443aa1ae90a4f6c4b742103ba80584598f2e" dependencies = [ "native-tls", "tokio", @@ -9493,25 +9961,25 @@ dependencies = [ [[package]] name = "postgres-protocol" -version = "0.6.9" -source = "git+https://github.com/MaterializeInc/rust-postgres?rev=78c1222577bb091d69bc22b1bc7ad01c14675abe#78c1222577bb091d69bc22b1bc7ad01c14675abe" +version = "0.6.12" +source = "git+https://github.com/windmill-labs/rust-postgres?rev=b3a6443aa1ae90a4f6c4b742103ba80584598f2e#b3a6443aa1ae90a4f6c4b742103ba80584598f2e" dependencies = [ "base64 0.22.1", "byteorder", "bytes", "fallible-iterator", - "hmac", - "md-5 0.10.6", + "hmac 0.13.0", + "md-5 0.11.0", "memchr", - "rand 0.9.5", - "sha2 0.10.9", + "rand 0.10.3", + "sha2 0.11.0", "stringprep", ] [[package]] name = "postgres-types" version = "0.2.11" -source = "git+https://github.com/MaterializeInc/rust-postgres?rev=78c1222577bb091d69bc22b1bc7ad01c14675abe#78c1222577bb091d69bc22b1bc7ad01c14675abe" +source = "git+https://github.com/windmill-labs/rust-postgres?rev=b3a6443aa1ae90a4f6c4b742103ba80584598f2e#b3a6443aa1ae90a4f6c4b742103ba80584598f2e" dependencies = [ "array-init", "bit-vec 0.6.3", @@ -9537,9 +10005,9 @@ dependencies = [ [[package]] name = "powerfmt" -version = "0.2.0" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" +checksum = "4a6394b9e965e73d0a289ee54f589087e2c676aedf60885baf52c76b771e4958" [[package]] name = "ppv-lite86" @@ -9582,7 +10050,7 @@ version = "0.13.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "353e1ca18966c16d9deb1c69278edbc5f194139612772bd9537af60ac231e1e6" dependencies = [ - "elliptic-curve", + "elliptic-curve 0.13.8", ] [[package]] @@ -9591,7 +10059,7 @@ version = "3.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e67ba7e9b2b56446f1d419b1d807906278ffa1a658a8a5d8a39dcb1f5a78614f" dependencies = [ - "toml_edit 0.25.15+spec-1.1.0", + "toml_edit 0.25.17+spec-1.1.0", ] [[package]] @@ -9617,29 +10085,6 @@ dependencies = [ "version_check", ] -[[package]] -name = "proc-macro-rules" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07c277e4e643ef00c1233393c673f655e3672cf7eb3ba08a00bdd0ea59139b5f" -dependencies = [ - "proc-macro-rules-macros", - "proc-macro2", - "syn 2.0.119", -] - -[[package]] -name = "proc-macro-rules-macros" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "207fffb0fe655d1d47f6af98cc2793405e85929bdbc420d685554ff07be27ac7" -dependencies = [ - "once_cell", - "proc-macro2", - "quote", - "syn 2.0.119", -] - [[package]] name = "proc-macro-utils" version = "0.10.0" @@ -9713,6 +10158,21 @@ dependencies = [ "thiserror 2.0.21", ] +[[package]] +name = "proptest" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b45fcc2344c680f5025fe57779faef368840d0bd1f42f216291f0dc4ace4744" +dependencies = [ + "bitflags 2.13.2", + "num-traits", + "rand 0.9.5", + "rand_chacha 0.9.0", + "rand_xorshift", + "regex-syntax 0.8.11", + "unarray", +] + [[package]] name = "prost" version = "0.13.5" @@ -9720,7 +10180,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2796faa41db3ec313a31f7624d9286acf277b52de526150b7e69f3debf891ee5" dependencies = [ "bytes", - "prost-derive", + "prost-derive 0.13.5", +] + +[[package]] +name = "prost" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "528ac67416ff8646872a3c02cad9cc4ee5dc9f9540c9b10771855c95cb2e5ae1" +dependencies = [ + "bytes", + "prost-derive 0.14.4", ] [[package]] @@ -9736,13 +10206,26 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "prost-derive" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b570b25f7617e43d59005d0990ccb79e950a423952cea19671b7a876da390adf" +dependencies = [ + "anyhow", + "itertools 0.14.0", + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "prost-types" version = "0.13.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "52c2c1bf36ddb1a1c396b3601a3cec27c2462e45f07c386894ec3ccf5332bd16" dependencies = [ - "prost", + "prost 0.13.5", ] [[package]] @@ -9860,7 +10343,7 @@ dependencies = [ "quinn-proto", "quinn-udp", "rustc-hash 2.1.3", - "rustls 0.23.35", + "rustls 0.23.40", "socket2 0.6.5", "thiserror 2.0.21", "tokio", @@ -9876,13 +10359,14 @@ checksum = "0e750cca55fe4f0439a15d0bb529da9651e79993e8e72c61a899a36d462befbe" dependencies = [ "aws-lc-rs", "bytes", + "fastbloom", "getrandom 0.4.3", "lru-slab", "rand 0.10.3", "rand_pcg", "ring 0.17.14", "rustc-hash 2.1.3", - "rustls 0.23.35", + "rustls 0.23.40", "rustls-pki-types", "slab", "thiserror 2.0.21", @@ -9914,6 +10398,28 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "quote-use" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48356e493823dc3c1d7e9fc7b3ff204ecec48afbdd718fc4134091b88eb63f3d" +dependencies = [ + "quote", + "quote-use-macros", +] + +[[package]] +name = "quote-use-macros" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8cf1c5de6a7cc231c71e03c77aa4c7019097859ea7029156b54d2894a8430704" +dependencies = [ + "proc-macro-utils", + "proc-macro2", + "quote", + "syn 3.0.6", +] + [[package]] name = "r-efi" version = "5.3.0" @@ -10068,6 +10574,15 @@ dependencies = [ "rand_core 0.10.1", ] +[[package]] +name = "rand_xorshift" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "513962919efc330f829edb2535844d1b912b0fbe2ca165d613e4e8788bb05a5a" +dependencies = [ + "rand_core 0.9.5", +] + [[package]] name = "raw-cpuid" version = "11.6.0" @@ -10304,9 +10819,9 @@ checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" [[package]] name = "relative-path" -version = "2.0.1" +version = "2.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bca40a312222d8ba74837cb474edef44b37f561da5f773981007a10bbaa992b0" +checksum = "f2a073568cd8b9c4825429d3e9fc5f52851b4f97cc1348800689408921e6b8f8" dependencies = [ "serde", ] @@ -10332,11 +10847,11 @@ dependencies = [ "futures-channel", "futures-core", "futures-util", - "h2 0.4.19", + "h2 0.4.20", "http 1.5.0", "http-body 1.1.0", "http-body-util", - "hyper 1.11.1", + "hyper 1.12.0", "hyper-rustls 0.27.10", "hyper-tls", "hyper-util", @@ -10347,7 +10862,7 @@ dependencies = [ "percent-encoding", "pin-project-lite", "quinn", - "rustls 0.23.35", + "rustls 0.23.40", "rustls-native-certs 0.8.4", "rustls-pki-types", "serde", @@ -10380,11 +10895,11 @@ dependencies = [ "encoding_rs", "futures-core", "futures-util", - "h2 0.4.19", + "h2 0.4.20", "http 1.5.0", "http-body 1.1.0", "http-body-util", - "hyper 1.11.1", + "hyper 1.12.0", "hyper-rustls 0.27.10", "hyper-util", "js-sys", @@ -10394,7 +10909,7 @@ dependencies = [ "percent-encoding", "pin-project-lite", "quinn", - "rustls 0.23.35", + "rustls 0.23.40", "rustls-pki-types", "rustls-platform-verifier", "serde", @@ -10440,7 +10955,7 @@ dependencies = [ "futures", "getrandom 0.2.17", "http 1.5.0", - "hyper 1.11.1", + "hyper 1.12.0", "reqwest 0.13.5", "reqwest-middleware", "retry-policies", @@ -10450,6 +10965,16 @@ dependencies = [ "wasmtimer", ] +[[package]] +name = "resb" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22d392791f3c6802a1905a509e9d1a6039cbbcb5e9e00e5a6d3661f7c874f390" +dependencies = [ + "potential_utf", + "serde_core", +] + [[package]] name = "resolv-conf" version = "0.7.6" @@ -10471,7 +10996,7 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8dd2a808d456c4a54e300a23e9f5a67e122c3024119acbfd73e3bf664491cb2" dependencies = [ - "hmac", + "hmac 0.12.1", "subtle", ] @@ -10569,9 +11094,9 @@ dependencies = [ [[package]] name = "rmcp-macros" -version = "3.5.0" +version = "3.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7e66877136e969d5a2d561ee5841b17717a0619aebd64abb47a2a2f46e34683" +checksum = "a1da498700030856ba63114bceefa30ba21f5e278cb975e5b9de3529fe842b40" dependencies = [ "darling 0.24.1", "proc-macro2", @@ -10634,20 +11159,120 @@ version = "0.9.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b8573f03f5883dcaebdfcf4725caa1ecb9c15b2ef50c43a07b816e06799bb12d" dependencies = [ - "const-oid", + "const-oid 0.9.6", "digest 0.10.7", "num-bigint-dig", "num-integer", "num-traits", "pkcs1", - "pkcs8", + "pkcs8 0.10.2", "rand_core 0.6.4", - "signature", - "spki", + "signature 2.2.0", + "spki 0.7.3", "subtle", "zeroize", ] +[[package]] +name = "ruff_python_ast" +version = "0.0.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15434b06df7d930fad2818639a4e0fec4af9c770121c45a96b72da7feaae9ead" +dependencies = [ + "aho-corasick", + "arrayvec", + "bitflags 2.13.2", + "char_str", + "compact_str 0.10.0", + "get-size2", + "is-macro", + "memchr", + "ruff_python_trivia", + "ruff_source_file", + "ruff_text_size", + "rustc-hash 2.1.3", + "thin-vec", + "thiserror 2.0.21", +] + +[[package]] +name = "ruff_python_parser" +version = "0.0.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37a22811d752aa9e255d7dd72524d14352cf9bc654ef79b399bf1eeeb6640fbb" +dependencies = [ + "bitflags 2.13.2", + "bstr", + "drop_bomb", + "get-size2", + "hashbrown 0.17.1", + "memchr", + "ruff_python_ast", + "ruff_python_trivia", + "ruff_text_size", + "ruff_unicode_names2", + "rustc-hash 2.1.3", + "stacker", + "static_assertions", + "thin-vec", + "unicode-ident", + "unicode-normalization", +] + +[[package]] +name = "ruff_python_trivia" +version = "0.0.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0322a5059fe0feb2e010ca6f31c24ac289f6b7e9e38359539491b10c57f1c5bc" +dependencies = [ + "itertools 0.15.0", + "memchr", + "ruff_source_file", + "ruff_text_size", + "rustc-hash 2.1.3", + "unicode-ident", + "unicode-normalization", +] + +[[package]] +name = "ruff_source_file" +version = "0.0.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6ae809a289b5c791f82aa45df33f2a92323e7a1c1d265c10e9ac884027a2e43" +dependencies = [ + "memchr", + "ruff_text_size", +] + +[[package]] +name = "ruff_text_size" +version = "0.0.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "daf6344d6915ced9316312c1ee3f24c4300c54295bc4c3b8f4e95b633ae5443b" +dependencies = [ + "get-size2", +] + +[[package]] +name = "ruff_unicode_names2" +version = "0.0.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13418b8fd029d597e35dedc36e91f47122d26075e4c86557574d9441753fb375" +dependencies = [ + "phf 0.11.3", + "ruff_unicode_names2_generator", +] + +[[package]] +name = "ruff_unicode_names2_generator" +version = "0.0.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37fd82af49d5685d9054e68185987acaaf62542f671b6df8935155664876bc39" +dependencies = [ + "phf_codegen", + "rand 0.8.5", +] + [[package]] name = "rumqttc" version = "0.24.0" @@ -10721,12 +11346,6 @@ dependencies = [ "wasm-bindgen", ] -[[package]] -name = "rustc-demangle" -version = "0.1.28" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b74b56ffa8bb2830709a538c2cbcae9aa062db0d2a42563bfb09bdaae44020eb" - [[package]] name = "rustc-hash" version = "1.1.0" @@ -10739,22 +11358,13 @@ version = "2.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d" -[[package]] -name = "rustc_version" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a" -dependencies = [ - "semver 0.9.0", -] - [[package]] name = "rustc_version" version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" dependencies = [ - "semver 1.0.28", + "semver", ] [[package]] @@ -10834,9 +11444,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.35" +version = "0.23.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "533f54bc6a7d4f647e46ad909549eda97bf5afc1585190ef692b4286b198bd8f" +checksum = "ef86cd5876211988985292b91c96a8f2d298df24e75989a43a3c73f2d4d8168b" dependencies = [ "aws-lc-rs", "log", @@ -10855,7 +11465,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "70cc376c6ba1823ae229bacf8ad93c136d93524eab0e4e5e0e4f96b9c4e5b212" dependencies = [ "log", - "rustls 0.23.35", + "rustls 0.23.40", "rustls-native-certs 0.7.3", "rustls-pki-types", "rustls-webpki 0.103.15", @@ -10937,7 +11547,7 @@ dependencies = [ "jni", "log", "once_cell", - "rustls 0.23.35", + "rustls 0.23.40", "rustls-native-certs 0.8.4", "rustls-platform-verifier-android", "rustls-webpki 0.103.15", @@ -10955,12 +11565,13 @@ checksum = "eec689c0bc40ff2458a5977b6619cb718087084a18e02a131c599b62d05e1a5f" [[package]] name = "rustls-tokio-stream" -version = "0.5.0" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "faa7dc7c991d9164e55bbf1558029eb5b84d32cc4d61a7df5b8641b2deedc4b3" +checksum = "0560d12c0d8c672f849197de91b9ee61f5bf9aa024c97aaeeb112ec2f6c347fd" dependencies = [ + "derive-io", "futures", - "rustls 0.23.35", + "rustls 0.23.40", "socket2 0.5.10", "tokio", ] @@ -10998,63 +11609,6 @@ dependencies = [ "untrusted 0.9.0", ] -[[package]] -name = "rustpython-ast" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4cdaf8ee5c1473b993b398c174641d3aa9da847af36e8d5eb8291930b72f31a5" -dependencies = [ - "is-macro", - "malachite-bigint", - "rustpython-parser-core", - "static_assertions", -] - -[[package]] -name = "rustpython-parser" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "868f724daac0caf9bd36d38caf45819905193a901e8f1c983345a68e18fb2abb" -dependencies = [ - "anyhow", - "is-macro", - "itertools 0.11.0", - "lalrpop-util", - "log", - "malachite-bigint", - "num-traits", - "phf 0.11.3", - "phf_codegen", - "rustc-hash 1.1.0", - "rustpython-ast", - "rustpython-parser-core", - "tiny-keccak", - "unic-emoji-char", - "unic-ucd-ident", - "unicode_names2", -] - -[[package]] -name = "rustpython-parser-core" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4b6c12fa273825edc7bccd9a734f0ad5ba4b8a2f4da5ff7efe946f066d0f4ad" -dependencies = [ - "is-macro", - "memchr", - "rustpython-parser-vendored", -] - -[[package]] -name = "rustpython-parser-vendored" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04fcea49a4630a3a5d940f4d514dc4f575ed63c14c3e3ed07146634aed7f67a6" -dependencies = [ - "memchr", - "once_cell", -] - [[package]] name = "rustversion" version = "1.0.23" @@ -11262,11 +11816,25 @@ version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3e97a565f76233a6003f9f5c54be1d9c5bdfa3eccfb189469f11ec4901c47dc" dependencies = [ - "base16ct", - "der", + "base16ct 0.2.0", + "der 0.7.10", "generic-array", - "pkcs8", - "serdect", + "pkcs8 0.10.2", + "serdect 0.2.0", + "subtle", + "zeroize", +] + +[[package]] +name = "sec1" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d56d437c2f19203ce5f7122e507831de96f3d2d4d3be5af44a0b0a09d8a80e4d" +dependencies = [ + "base16ct 1.0.0", + "ctutils", + "der 0.8.2", + "hybrid-array", "subtle", "zeroize", ] @@ -11316,27 +11884,12 @@ dependencies = [ "libc", ] -[[package]] -name = "semver" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403" -dependencies = [ - "semver-parser", -] - [[package]] name = "semver" version = "1.0.28" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" -[[package]] -name = "semver-parser" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" - [[package]] name = "seq-macro" version = "0.3.6" @@ -11515,16 +12068,16 @@ dependencies = [ [[package]] name = "serde_v8" -version = "0.261.1" +version = "0.321.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3495190857461e87a2716141043218aad5281f219f54a03b7ebbe605b3b931df" +checksum = "fd3b6f6140192f569b88997e2ea7afeef6aa38b64374e84784ea1d1ea35679d3" dependencies = [ - "deno_error 0.6.1", + "deno_error", + "deno_v8", "num-bigint", "serde", "smallvec", "thiserror 2.0.21", - "v8", ] [[package]] @@ -11594,7 +12147,17 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a84f14a19e9a014bb9f4512488d9829a68e04ecabffb0f9904cd1ace94598177" dependencies = [ - "base16ct", + "base16ct 0.2.0", + "serde", +] + +[[package]] +name = "serdect" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66cf8fedced2fcf12406bcb34223dffb92eaf34908ede12fed414c82b7f00b3e" +dependencies = [ + "base16ct 1.0.0", "serde", ] @@ -11658,6 +12221,38 @@ dependencies = [ "digest 0.10.7", ] +[[package]] +name = "sha2" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "446ba717509524cb3f22f17ecc096f10f4822d76ab5c0b9822c5f9c284e825f4" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.1", + "digest 0.11.3", +] + +[[package]] +name = "sha3" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77fd7028345d415a4034cf8777cd4f8ab1851274233b45f84e3d955502d93874" +dependencies = [ + "digest 0.10.7", + "keccak 0.1.6", +] + +[[package]] +name = "shake" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09057cb2149ad4cbd2da1e26b351f9a4c354219421229c69c3063e6f61947c4a" +dependencies = [ + "digest 0.11.3", + "keccak 0.2.2", + "sponge-cursor", +] + [[package]] name = "sharded-slab" version = "0.1.7" @@ -11698,6 +12293,16 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" +[[package]] +name = "signal-hook" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d881a16cf4426aa584979d30bd82cb33429027e42122b169753d6ef1085ed6e2" +dependencies = [ + "libc", + "signal-hook-registry", +] + [[package]] name = "signal-hook-registry" version = "1.4.8" @@ -11714,9 +12319,9 @@ version = "0.27.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c1e303f8205714074f6068773f0e29527e0453937fe837c9717d066635b65f31" dependencies = [ - "pkcs8", + "pkcs8 0.10.2", "rand_core 0.6.4", - "signature", + "signature 2.2.0", "zeroize", ] @@ -11730,6 +12335,16 @@ dependencies = [ "rand_core 0.6.4", ] +[[package]] +name = "signature" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28d567dcbaf0049cb8ac2608a76cd95ff9e4412e1899d389ee400918ca7537f5" +dependencies = [ + "digest 0.11.3", + "rand_core 0.10.1", +] + [[package]] name = "simd-adler32" version = "0.3.10" @@ -11742,7 +12357,7 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "11031e251abf8611c80f460e19dbdeb54a66db918e49c65a7065b46ac7aec520" dependencies = [ - "rustc_version 0.4.1", + "rustc_version", "simdutf8", ] @@ -11910,7 +12525,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d" dependencies = [ "base64ct", - "der", + "der 0.7.10", +] + +[[package]] +name = "spki" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ef958a98b9d5da290cfc78946e9f3e61e1e62a18db0d92cac0b83cc161491a9" +dependencies = [ + "base64ct", + "der 0.8.2", ] [[package]] @@ -11925,6 +12550,12 @@ dependencies = [ "unicode-segmentation", ] +[[package]] +name = "sponge-cursor" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a0219bd7d979d58245a4f41f695e1ac9f8befdffadd7f61f1bae9e39abc6620" + [[package]] name = "sql-builder" version = "3.1.1" @@ -12004,7 +12635,7 @@ dependencies = [ "memchr", "once_cell", "percent-encoding", - "rustls 0.23.35", + "rustls 0.23.40", "serde", "serde_json", "sha2 0.10.9", @@ -12077,7 +12708,7 @@ dependencies = [ "generic-array", "hex", "hkdf", - "hmac", + "hmac 0.12.1", "itoa", "log", "md-5 0.10.6", @@ -12117,7 +12748,7 @@ dependencies = [ "futures-util", "hex", "hkdf", - "hmac", + "hmac 0.12.1", "home", "itoa", "log", @@ -12201,6 +12832,15 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" +[[package]] +name = "strck" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42316e70da376f3d113a68d138a60d8a9883c604fe97942721ec2068dab13a9f" +dependencies = [ + "unicode-ident", +] + [[package]] name = "string_enum" version = "1.0.2" @@ -12401,7 +13041,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bcf55c2d7555c93f4945e29f93b7529562be97ba16e60dd94c25724d746174ac" dependencies = [ "ascii", - "compact_str", + "compact_str 0.7.1", "memchr", "num-bigint", "once_cell", @@ -12714,6 +13354,17 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "syn-match" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "54b8f0a9004d6aafa6a588602a1119e6cdaacec9921aa1605383e6e7d6258fd6" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "sync_wrapper" version = "1.0.2" @@ -12723,6 +13374,18 @@ dependencies = [ "futures-core", ] +[[package]] +name = "synstructure" +version = "0.12.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f36bdaa60a83aca3921b5259d5400cbf5e90fc51931376a9bd4a0eb79aa7210f" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", + "unicode-xid", +] + [[package]] name = "synstructure" version = "0.13.2" @@ -12756,11 +13419,14 @@ dependencies = [ [[package]] name = "sys_traits" -version = "0.1.16" +version = "0.1.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc4707edf3196e8037ee45018d1bb1bfb233b0e4fc440fa3d3f25bc69bfdaf26" +checksum = "c7a35f11f9773f11930a7d1e6276766f1577400ee2530b7f0cf3844c2dcbf12b" dependencies = [ + "junction", + "libc", "sys_traits_macros", + "windows-sys 0.59.0", ] [[package]] @@ -13015,15 +13681,6 @@ dependencies = [ "rustls-pemfile 2.2.0", ] -[[package]] -name = "temp_deno_which" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "366c5ccd670145885feb6efd6bbf2478ed236c4c3839046fcc8e2a1a84c51091" -dependencies = [ - "either", -] - [[package]] name = "tempfile" version = "3.27.0" @@ -13037,6 +13694,40 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "temporal_capi" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f48eb9ca4ca22caaa48d1e7a534ad3fd598a23d1a71945e8ad5ac0be501fa5f2" +dependencies = [ + "diplomat", + "diplomat-runtime", + "icu_calendar", + "icu_locale_core", + "num-traits", + "temporal_rs", + "timezone_provider", + "writeable", + "zoneinfo64", +] + +[[package]] +name = "temporal_rs" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a978fef907a27736827d336c0bfaab14fb8a7509dd6dc2b43c44e51d59aca5b" +dependencies = [ + "calendrical_calculations", + "core_maths", + "icu_calendar", + "icu_locale_core", + "ixdtf", + "num-traits", + "timezone_provider", + "tinystr", + "writeable", +] + [[package]] name = "termcolor" version = "1.4.1" @@ -13075,6 +13766,12 @@ dependencies = [ "unicode-width 0.2.2", ] +[[package]] +name = "thin-vec" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6a4b9ba8738cb4a4f399d37e266becfd475e75eb73425b87a05a2f2039ba63e" + [[package]] name = "thiserror" version = "1.0.69" @@ -13237,6 +13934,18 @@ dependencies = [ "time-core", ] +[[package]] +name = "timezone_provider" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89ec4f4eebb4817fde02a411aedc7ac5f66c89c68c49e5d4b17a8139f077af52" +dependencies = [ + "tinystr", + "zerotrie", + "zerovec", + "zoneinfo64", +] + [[package]] name = "tiny-keccak" version = "2.0.2" @@ -13313,23 +14022,20 @@ dependencies = [ [[package]] name = "tokio" -version = "1.46.1" +version = "1.53.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0cc3a2344dafbe23a245241fe8b09735b521110d30fcefbbd5feb1797ca35d17" +checksum = "e95f91fcc7a621e8b030f6aa23c71fe9838ae2fb4d8118b75602a328f5144044" dependencies = [ - "backtrace", "bytes", - "io-uring", "libc", "mio", "parking_lot", "pin-project-lite", "signal-hook-registry", - "slab", - "socket2 0.5.10", + "socket2 0.6.5", "tokio-macros", "tracing", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -13358,13 +14064,13 @@ dependencies = [ [[package]] name = "tokio-macros" -version = "2.5.0" +version = "2.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e06d43f1345a3bcd39f6a56dbb7dcab2ba47e68e8ac134855e7e2bdbaf8cab8" +checksum = "78773a2a397f451582ce068015985c33193cf6dea8b74d2a639fe457b2f07b0e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn 3.0.6", ] [[package]] @@ -13380,7 +14086,7 @@ dependencies = [ [[package]] name = "tokio-postgres" version = "0.7.15" -source = "git+https://github.com/MaterializeInc/rust-postgres?rev=78c1222577bb091d69bc22b1bc7ad01c14675abe#78c1222577bb091d69bc22b1bc7ad01c14675abe" +source = "git+https://github.com/windmill-labs/rust-postgres?rev=b3a6443aa1ae90a4f6c4b742103ba80584598f2e#b3a6443aa1ae90a4f6c4b742103ba80584598f2e" dependencies = [ "async-trait", "byteorder", @@ -13453,7 +14159,7 @@ version = "0.26.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c9cc2678c2cdd569ef8215e2afd7954ada2ae20b4fdd2c5fe6139a3b02d105db" dependencies = [ - "rustls 0.23.35", + "rustls 0.23.40", "tokio", ] @@ -13585,9 +14291,9 @@ dependencies = [ [[package]] name = "toml_datetime" -version = "1.1.1+spec-1.1.0" +version = "1.1.2+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3165f65f62e28e0115a00b2ebdd37eb6f3b641855f9d636d3cd4103767159ad7" +checksum = "2b86d767906c6c42421dcba507eb9d203e779497710a47782a224bb871653053" dependencies = [ "serde_core", ] @@ -13607,21 +14313,21 @@ dependencies = [ [[package]] name = "toml_edit" -version = "0.25.15+spec-1.1.0" +version = "0.25.17+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1340ea94a5856333492c9064b02c778b191dd2c853778d9609debdcdfea3a614" +checksum = "e3641d5bbb5349a79e1020a242d251efbc546ad8048d133958323ce9c40a9c9c" dependencies = [ "indexmap 2.14.2", - "toml_datetime 1.1.1+spec-1.1.0", + "toml_datetime 1.1.2+spec-1.1.0", "toml_parser", "winnow 1.0.4", ] [[package]] name = "toml_parser" -version = "1.1.3+spec-1.1.0" +version = "1.1.5+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d38ac1cf9b95face32296c0a3ede1fdc270627c9d9c02a7274dd6d960dc4d56" +checksum = "baa693a8032d7e1cada7d0041e96126df243179ff061456783ac7f12bda4744c" dependencies = [ "winnow 1.0.4", ] @@ -13632,24 +14338,20 @@ version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "877c5b330756d856ffcc4553ab34a5684481ade925ecc54bcd1bf02b1d0d4d52" dependencies = [ - "async-stream", "async-trait", - "axum 0.7.9", "base64 0.22.1", "bytes", "flate2", - "h2 0.4.19", "http 1.5.0", "http-body 1.1.0", "http-body-util", - "hyper 1.11.1", + "hyper 1.12.0", "hyper-timeout", "hyper-util", "percent-encoding", "pin-project", - "prost", + "prost 0.13.5", "rustls-pemfile 2.2.0", - "socket2 0.5.10", "tokio", "tokio-rustls 0.26.6", "tokio-stream", @@ -13667,20 +14369,20 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7e581ba15a835f4d9ea06c55ab1bd4dce26fc53752c69a04aac00703bfb49ba9" dependencies = [ "async-trait", - "axum 0.8.9", + "axum", "base64 0.22.1", "bytes", "flate2", - "h2 0.4.19", + "h2 0.4.20", "http 1.5.0", "http-body 1.1.0", "http-body-util", - "hyper 1.11.1", + "hyper 1.12.0", "hyper-timeout", "hyper-util", "percent-encoding", "pin-project", - "prost", + "prost 0.13.5", "rustls-native-certs 0.8.4", "socket2 0.5.10", "tokio", @@ -13738,7 +14440,7 @@ version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "151b5a3e3c45df17466454bb74e9ecedecc955269bdedbf4d150dfa393b55a36" dependencies = [ - "axum-core 0.5.6", + "axum-core", "cookie", "futures-util", "http 1.5.0", @@ -13936,9 +14638,9 @@ dependencies = [ [[package]] name = "tree-sitter-language" -version = "0.1.8" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca0d1bf6fdd806e43ae5198f82f527056d359def39e54e67a0f478ac09dac081" +checksum = "d0af592be68c579aa78a16846bd19422978c3c52e438523d45ff5d1bff1f9d4a" [[package]] name = "tree-sitter-r" @@ -14029,9 +14731,9 @@ dependencies = [ [[package]] name = "twox-hash" -version = "2.1.4" +version = "2.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5283634e518fe9e82c7b20520bb4bc209009fd16c82077c802f8111ecbb0117a" +checksum = "86a801b3cea342a06d468c8710662aa29e5e05e4f5c0d62f00bbb7f2ad7941c2" [[package]] name = "typed-path" @@ -14092,6 +14794,12 @@ dependencies = [ "web-time", ] +[[package]] +name = "unarray" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eaea85b334db583fe3274d12b4cd1880032beab409c0d774be044d4480ab9a94" + [[package]] name = "unic-char-property" version = "0.9.0" @@ -14113,17 +14821,6 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "80d7ff825a6a654ee85a63e80f92f054f904f21e7d12da4e22f9834a4aaa35bc" -[[package]] -name = "unic-emoji-char" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b07221e68897210270a38bde4babb655869637af0f69407f96053a34f76494d" -dependencies = [ - "unic-char-property", - "unic-char-range", - "unic-ucd-version", -] - [[package]] name = "unic-ucd-category" version = "0.9.0" @@ -14136,17 +14833,6 @@ dependencies = [ "unic-ucd-version", ] -[[package]] -name = "unic-ucd-ident" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e230a37c0381caa9219d67cf063aa3a375ffed5bf541a452db16e744bdab6987" -dependencies = [ - "unic-char-property", - "unic-char-range", - "unic-ucd-version", -] - [[package]] name = "unic-ucd-version" version = "0.9.0" @@ -14158,9 +14844,9 @@ dependencies = [ [[package]] name = "unicase" -version = "2.9.0" +version = "2.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dbc4bc3a9f746d862c45cb89d705aa10f187bb96c76001afab07a0d35ce60142" +checksum = "357cc3acc6a036009fd6c973ed009037c732d60d0b4f6c673e9041497482a28f" [[package]] name = "unicode-bidi" @@ -14234,28 +14920,6 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e" -[[package]] -name = "unicode_names2" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1673eca9782c84de5f81b82e4109dcfb3611c8ba0d52930ec4a9478f547b2dd" -dependencies = [ - "phf 0.11.3", - "unicode_names2_generator", -] - -[[package]] -name = "unicode_names2_generator" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b91e5b84611016120197efd7dc93ef76774f4e084cd73c9fb3ea4a86c570c56e" -dependencies = [ - "getopts", - "log", - "phf_codegen", - "rand 0.8.5", -] - [[package]] name = "universal-hash" version = "0.5.1" @@ -14307,7 +14971,7 @@ dependencies = [ "log", "native-tls", "once_cell", - "rustls 0.23.35", + "rustls 0.23.40", "rustls-pki-types", "serde", "serde_json", @@ -14337,13 +15001,13 @@ checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da" [[package]] name = "urlpattern" -version = "0.3.0" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70acd30e3aa1450bc2eece896ce2ad0d178e9c079493819301573dae3c37ba6d" +checksum = "0f805818f843b548bacc19609eb3619dd2850e54746f5cada37927393c2ef4ec" dependencies = [ + "icu_properties", "regex", "serde", - "unic-ucd-ident", "url", ] @@ -14379,9 +15043,9 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] name = "uuid" -version = "1.26.1" +version = "1.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ef6dac1e96601b4fb3acccccff2139741fcb757cb9a36089bf5be91cfb285ce" +checksum = "97277d36b9c3ace13e58fa6e753f8b0bbbf302a18dd193240d70f9e29681059a" dependencies = [ "getrandom 0.4.3", "js-sys", @@ -14391,20 +15055,36 @@ dependencies = [ [[package]] name = "v8" -version = "137.3.0" +version = "150.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33995a1fee055ff743281cde33a41f0d618ee0bdbe8bdf6859e11864499c2595" +checksum = "42a978ff11f15b24e5c05a7123cf2b68f41e763546699781a924ef4e2cf43a49" dependencies = [ - "bindgen 0.71.1", + "bindgen 0.72.1", "bitflags 2.13.2", "fslock", "gzip-header", "home", "miniz_oxide 0.8.9", "paste", + "temporal_capi", "which 6.0.3", ] +[[package]] +name = "v8x" +version = "149.4.0-rc.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3701ffb3692a24808503d17f758b0a3f688d2c451779247982566b5941033e63" +dependencies = [ + "bindgen 0.70.1", + "bitflags 2.13.2", + "cc", + "paste", + "temporal_capi", + "temporal_rs", + "timezone_provider", +] + [[package]] name = "valuable" version = "0.1.1" @@ -14466,9 +15146,9 @@ dependencies = [ [[package]] name = "want" -version = "0.3.1" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +checksum = "ec4cdd0dd910afe868b7ef477227d8d538b46b3075031afee8a9f2acb0a2ed0b" dependencies = [ "try-lock", ] @@ -14624,11 +15304,11 @@ dependencies = [ [[package]] name = "wasm_dep_analyzer" -version = "0.3.0" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e51cf5f08b357e64cd7642ab4bbeb11aecab9e15520692129624fb9908b8df2c" +checksum = "a10e6b67c951a84de7029487e0e0a496860dae49f6699edd279d5ff35b8fbf54" dependencies = [ - "deno_error 0.6.1", + "deno_error", "thiserror 2.0.21", ] @@ -14729,6 +15409,12 @@ dependencies = [ "winsafe", ] +[[package]] +name = "which" +version = "8.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bae2f2b2b816647a1cab1acc91f5bd20812d53cb344382635ec2181940c8034f" + [[package]] name = "whoami" version = "1.6.1" @@ -14792,14 +15478,14 @@ dependencies = [ [[package]] name = "windmill" -version = "1.821.0" +version = "1.830.0" dependencies = [ "anyhow", "async-nats", "aws-config", "aws-credential-types", "aws-sdk-sqs", - "axum 0.8.9", + "axum", "base64 0.22.1", "chrono", "constant_time_eq 0.3.1", @@ -14808,7 +15494,7 @@ dependencies = [ "gethostname", "git-version", "hex", - "hmac", + "hmac 0.12.1", "jsonwebtoken 10.4.0", "lazy_static", "once_cell", @@ -14819,7 +15505,7 @@ dependencies = [ "rdkafka", "reqwest 0.13.5", "rumqttc", - "rustls 0.23.35", + "rustls 0.23.40", "serde", "serde_derive", "serde_json", @@ -14850,6 +15536,7 @@ dependencies = [ "windmill-api-scripts", "windmill-api-settings", "windmill-api-users", + "windmill-audit", "windmill-autoscaling", "windmill-common", "windmill-dep-map", @@ -14881,7 +15568,7 @@ dependencies = [ [[package]] name = "windmill-ai" -version = "1.821.0" +version = "1.830.0" dependencies = [ "async-stream", "async-trait", @@ -14898,6 +15585,7 @@ dependencies = [ "indexmap 2.14.2", "lazy_static", "mime_guess", + "rand 0.9.5", "reqwest 0.13.5", "serde", "serde_json", @@ -14915,9 +15603,9 @@ dependencies = [ [[package]] name = "windmill-alerting" -version = "1.821.0" +version = "1.830.0" dependencies = [ - "axum 0.8.9", + "axum", "chrono", "serde", "serde_json", @@ -14928,7 +15616,7 @@ dependencies = [ [[package]] name = "windmill-api" -version = "1.821.0" +version = "1.830.0" dependencies = [ "anyhow", "argon2", @@ -14939,7 +15627,7 @@ dependencies = [ "async_zip", "aws-sdk-config", "aws-sigv4", - "axum 0.8.9", + "axum", "base32", "base64 0.22.1", "bytes", @@ -14956,9 +15644,9 @@ dependencies = [ "futures", "git-version", "hex", - "hmac", + "hmac 0.12.1", "http 1.5.0", - "hyper 1.11.1", + "hyper 1.12.0", "indexmap 2.14.2", "itertools 0.14.0", "jsonwebtoken 10.4.0", @@ -14971,17 +15659,19 @@ dependencies = [ "native-tls", "openidconnect", "openssl", + "p256", "pin-project", "postgres-native-tls 0.5.3", "prometheus", "quick-xml", "quick_cache", "rand 0.9.5", + "rcgen", "regex", "reqwest 0.13.5", "rsa", "rust-embed", - "rustls 0.23.35", + "rustls 0.23.40", "samael", "serde", "serde_json", @@ -14997,6 +15687,7 @@ dependencies = [ "tokio", "tokio-native-tls", "tokio-postgres", + "tokio-rustls 0.26.6", "tokio-stream", "tokio-util", "tower 0.5.3", @@ -15068,12 +15759,12 @@ dependencies = [ [[package]] name = "windmill-api-agent-workers" -version = "1.821.0" +version = "1.830.0" dependencies = [ - "axum 0.8.9", + "axum", "chrono", "http 1.5.0", - "hyper 1.11.1", + "hyper 1.12.0", "lazy_static", "quick_cache", "serde", @@ -15091,9 +15782,9 @@ dependencies = [ [[package]] name = "windmill-api-assets" -version = "1.821.0" +version = "1.830.0" dependencies = [ - "axum 0.8.9", + "axum", "chrono", "serde", "serde_json", @@ -15108,10 +15799,10 @@ dependencies = [ [[package]] name = "windmill-api-auth" -version = "1.821.0" +version = "1.830.0" dependencies = [ "anyhow", - "axum 0.8.9", + "axum", "chrono", "http 1.5.0", "itertools 0.14.0", @@ -15134,7 +15825,7 @@ dependencies = [ [[package]] name = "windmill-api-client" -version = "1.821.0" +version = "1.830.0" dependencies = [ "reqwest 0.12.28", "serde", @@ -15144,9 +15835,9 @@ dependencies = [ [[package]] name = "windmill-api-configs" -version = "1.821.0" +version = "1.830.0" dependencies = [ - "axum 0.8.9", + "axum", "chrono", "itertools 0.14.0", "serde", @@ -15161,9 +15852,9 @@ dependencies = [ [[package]] name = "windmill-api-debug" -version = "1.821.0" +version = "1.830.0" dependencies = [ - "axum 0.8.9", + "axum", "base64 0.22.1", "chrono", "ed25519-dalek", @@ -15183,10 +15874,10 @@ dependencies = [ [[package]] name = "windmill-api-embeddings" -version = "1.821.0" +version = "1.830.0" dependencies = [ "anyhow", - "axum 0.8.9", + "axum", "candle-core", "candle-nn", "candle-transformers", @@ -15206,9 +15897,9 @@ dependencies = [ [[package]] name = "windmill-api-flow-conversations" -version = "1.821.0" +version = "1.830.0" dependencies = [ - "axum 0.8.9", + "axum", "chrono", "serde", "sql-builder", @@ -15222,11 +15913,11 @@ dependencies = [ [[package]] name = "windmill-api-flows" -version = "1.821.0" +version = "1.830.0" dependencies = [ - "axum 0.8.9", + "axum", "chrono", - "hyper 1.11.1", + "hyper 1.12.0", "serde", "serde_json", "sql-builder", @@ -15244,9 +15935,9 @@ dependencies = [ [[package]] name = "windmill-api-groups" -version = "1.821.0" +version = "1.830.0" dependencies = [ - "axum 0.8.9", + "axum", "chrono", "globset", "lazy_static", @@ -15266,9 +15957,9 @@ dependencies = [ [[package]] name = "windmill-api-inputs" -version = "1.821.0" +version = "1.830.0" dependencies = [ - "axum 0.8.9", + "axum", "chrono", "serde", "serde_json", @@ -15280,18 +15971,18 @@ dependencies = [ [[package]] name = "windmill-api-integration-tests" -version = "1.821.0" +version = "1.830.0" dependencies = [ "anyhow", "async-nats", "aws-config", "aws-credential-types", "aws-sdk-sqs", - "axum 0.8.9", + "axum", "base64 0.22.1", "futures", "hex", - "hmac", + "hmac 0.12.1", "rand 0.9.5", "rdkafka", "reqwest 0.13.5", @@ -15316,14 +16007,14 @@ dependencies = [ [[package]] name = "windmill-api-jobs" -version = "1.821.0" +version = "1.830.0" dependencies = [ "anyhow", - "axum 0.8.9", + "axum", "base64 0.22.1", "chrono", "http 1.5.0", - "hyper 1.11.1", + "hyper 1.12.0", "lazy_static", "serde", "serde_json", @@ -15335,15 +16026,16 @@ dependencies = [ "uuid", "windmill-api-auth", "windmill-api-sse", + "windmill-audit", "windmill-common", "windmill-queue", ] [[package]] name = "windmill-api-npm-proxy" -version = "1.821.0" +version = "1.830.0" dependencies = [ - "axum 0.8.9", + "axum", "chrono", "flate2", "futures", @@ -15351,7 +16043,7 @@ dependencies = [ "lazy_static", "quick_cache", "reqwest 0.13.5", - "semver 1.0.28", + "semver", "serde", "serde_json", "sha2 0.10.9", @@ -15369,10 +16061,10 @@ dependencies = [ [[package]] name = "windmill-api-openapi" -version = "1.821.0" +version = "1.830.0" dependencies = [ "anyhow", - "axum 0.8.9", + "axum", "http 1.5.0", "indexmap 2.14.2", "itertools 0.14.0", @@ -15391,9 +16083,9 @@ dependencies = [ [[package]] name = "windmill-api-schedule" -version = "1.821.0" +version = "1.830.0" dependencies = [ - "axum 0.8.9", + "axum", "chrono", "chrono-tz", "serde", @@ -15411,14 +16103,14 @@ dependencies = [ [[package]] name = "windmill-api-scripts" -version = "1.821.0" +version = "1.830.0" dependencies = [ "anyhow", - "axum 0.8.9", + "axum", "chrono", "futures", "http 1.5.0", - "hyper 1.11.1", + "hyper 1.12.0", "itertools 0.14.0", "lazy_static", "prometheus", @@ -15453,10 +16145,10 @@ dependencies = [ [[package]] name = "windmill-api-settings" -version = "1.821.0" +version = "1.830.0" dependencies = [ "anyhow", - "axum 0.8.9", + "axum", "base64 0.22.1", "bytes", "chrono", @@ -15482,7 +16174,7 @@ dependencies = [ [[package]] name = "windmill-api-sse" -version = "1.821.0" +version = "1.830.0" dependencies = [ "lazy_static", "serde", @@ -15494,13 +16186,13 @@ dependencies = [ [[package]] name = "windmill-api-users" -version = "1.821.0" +version = "1.830.0" dependencies = [ "argon2", - "axum 0.8.9", + "axum", "chrono", "http 1.5.0", - "hyper 1.11.1", + "hyper 1.12.0", "lazy_static", "serde", "serde_json", @@ -15518,9 +16210,9 @@ dependencies = [ [[package]] name = "windmill-api-workers" -version = "1.821.0" +version = "1.830.0" dependencies = [ - "axum 0.8.9", + "axum", "chrono", "serde", "serde_json", @@ -15532,15 +16224,15 @@ dependencies = [ [[package]] name = "windmill-api-workspaces" -version = "1.821.0" +version = "1.830.0" dependencies = [ - "axum 0.8.9", + "axum", "chrono", "constant_time_eq 0.3.1", "futures", "hex", "http 1.5.0", - "hyper 1.11.1", + "hyper 1.12.0", "lazy_static", "magic-crypt", "regex", @@ -15571,7 +16263,7 @@ dependencies = [ [[package]] name = "windmill-audit" -version = "1.821.0" +version = "1.830.0" dependencies = [ "chrono", "lazy_static", @@ -15580,15 +16272,16 @@ dependencies = [ "sql-builder", "sqlx", "tracing", + "tracing-subscriber", "windmill-common", ] [[package]] name = "windmill-autoscaling" -version = "1.821.0" +version = "1.830.0" dependencies = [ "anyhow", - "axum 0.8.9", + "axum", "k8s-openapi", "kube", "serde", @@ -15604,7 +16297,7 @@ dependencies = [ [[package]] name = "windmill-common" -version = "1.821.0" +version = "1.830.0" dependencies = [ "aes-gcm", "aho-corasick", @@ -15618,7 +16311,7 @@ dependencies = [ "aws-sdk-secretsmanager", "aws-sdk-sts", "aws-smithy-types-convert", - "axum 0.8.9", + "axum", "backon", "base64 0.22.1", "bitflags 2.13.2", @@ -15640,8 +16333,9 @@ dependencies = [ "git-version", "globset", "hex", - "hmac", - "hyper 1.11.1", + "hkdf", + "hmac 0.12.1", + "hyper 1.12.0", "indexmap 2.14.2", "itertools 0.14.0", "jsonwebtoken 10.4.0", @@ -15657,6 +16351,7 @@ dependencies = [ "opentelemetry-otlp 0.30.0", "opentelemetry-semantic-conventions 0.30.0", "opentelemetry_sdk 0.30.0", + "p256", "pep440_rs", "phf 0.11.3", "pin-project-lite", @@ -15672,17 +16367,18 @@ dependencies = [ "reqwest-middleware", "reqwest-retry", "rsa", - "rustls 0.23.35", + "rustls 0.23.40", "rustls-native-certs 0.8.4", + "ryu-js", "schemars 0.8.22", - "semver 1.0.28", + "semver", "serde", "serde_json", "serde_yml", "sha1", "sha2 0.10.9", "size", - "spki", + "spki 0.7.3", "sqlx", "strum", "strum_macros", @@ -15717,7 +16413,7 @@ dependencies = [ [[package]] name = "windmill-dep-map" -version = "1.821.0" +version = "1.830.0" dependencies = [ "chrono", "futures", @@ -15737,7 +16433,7 @@ dependencies = [ [[package]] name = "windmill-git-sync" -version = "1.821.0" +version = "1.830.0" dependencies = [ "regex", "serde", @@ -15754,7 +16450,7 @@ dependencies = [ [[package]] name = "windmill-indexer" -version = "1.821.0" +version = "1.830.0" dependencies = [ "anyhow", "astral-tokio-tar", @@ -15781,7 +16477,7 @@ dependencies = [ [[package]] name = "windmill-jseval" -version = "1.821.0" +version = "1.830.0" dependencies = [ "anyhow", "futures", @@ -15798,7 +16494,7 @@ dependencies = [ [[package]] name = "windmill-macros" -version = "1.821.0" +version = "1.830.0" dependencies = [ "itertools 0.14.0", "lazy_static", @@ -15814,7 +16510,7 @@ dependencies = [ [[package]] name = "windmill-mcp" -version = "1.821.0" +version = "1.830.0" dependencies = [ "anyhow", "async-trait", @@ -15835,15 +16531,15 @@ dependencies = [ [[package]] name = "windmill-native-triggers" -version = "1.821.0" +version = "1.830.0" dependencies = [ "anyhow", "async-trait", - "axum 0.8.9", + "axum", "backon", "base64 0.22.1", "chrono", - "hmac", + "hmac 0.12.1", "http 1.5.0", "itertools 0.14.0", "lazy_static", @@ -15866,16 +16562,16 @@ dependencies = [ [[package]] name = "windmill-oauth" -version = "1.821.0" +version = "1.830.0" dependencies = [ "anyhow", "arc-swap", "async-oauth2", - "axum 0.8.9", + "axum", "base64 0.22.1", "chrono", "hex", - "hmac", + "hmac 0.12.1", "itertools 0.14.0", "lazy_static", "reqwest 0.12.28", @@ -15891,7 +16587,7 @@ dependencies = [ [[package]] name = "windmill-object-store" -version = "1.821.0" +version = "1.830.0" dependencies = [ "anyhow", "async-stream", @@ -15900,7 +16596,7 @@ dependencies = [ "aws-credential-types", "aws-sdk-sts", "aws-smithy-types-convert", - "axum 0.8.9", + "axum", "bytes", "chrono", "datafusion", @@ -15927,7 +16623,7 @@ dependencies = [ [[package]] name = "windmill-operator" -version = "1.821.0" +version = "1.830.0" dependencies = [ "anyhow", "futures", @@ -15940,12 +16636,13 @@ dependencies = [ "thiserror 2.0.21", "tokio", "tracing", + "windmill-audit", "windmill-common", ] [[package]] name = "windmill-parser" -version = "1.821.0" +version = "1.830.0" dependencies = [ "convert_case 0.6.0", "serde", @@ -15954,7 +16651,7 @@ dependencies = [ [[package]] name = "windmill-parser-bash" -version = "1.821.0" +version = "1.830.0" dependencies = [ "anyhow", "lazy_static", @@ -15966,7 +16663,7 @@ dependencies = [ [[package]] name = "windmill-parser-csharp" -version = "1.821.0" +version = "1.830.0" dependencies = [ "anyhow", "serde_json", @@ -15978,7 +16675,7 @@ dependencies = [ [[package]] name = "windmill-parser-go" -version = "1.821.0" +version = "1.830.0" dependencies = [ "anyhow", "gosyn", @@ -15990,7 +16687,7 @@ dependencies = [ [[package]] name = "windmill-parser-graphql" -version = "1.821.0" +version = "1.830.0" dependencies = [ "anyhow", "lazy_static", @@ -16002,7 +16699,7 @@ dependencies = [ [[package]] name = "windmill-parser-java" -version = "1.821.0" +version = "1.830.0" dependencies = [ "anyhow", "serde_json", @@ -16014,7 +16711,7 @@ dependencies = [ [[package]] name = "windmill-parser-nu" -version = "1.821.0" +version = "1.830.0" dependencies = [ "anyhow", "nu-parser", @@ -16025,7 +16722,7 @@ dependencies = [ [[package]] name = "windmill-parser-php" -version = "1.821.0" +version = "1.830.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -16036,30 +16733,30 @@ dependencies = [ [[package]] name = "windmill-parser-py" -version = "1.821.0" +version = "1.830.0" dependencies = [ "anyhow", "itertools 0.14.0", - "rustpython-ast", - "rustpython-parser", + "ruff_python_ast", + "ruff_python_parser", "serde_json", "windmill-parser", ] [[package]] name = "windmill-parser-py-asset" -version = "1.821.0" +version = "1.830.0" dependencies = [ "anyhow", - "rustpython-ast", - "rustpython-parser", + "ruff_python_ast", + "ruff_python_parser", "windmill-parser", "windmill-parser-sql-asset", ] [[package]] name = "windmill-parser-py-imports" -version = "1.821.0" +version = "1.830.0" dependencies = [ "anyhow", "async-recursion", @@ -16069,7 +16766,8 @@ dependencies = [ "phf 0.11.3", "regex", "regex-lite", - "rustpython-parser", + "ruff_python_ast", + "ruff_python_parser", "serde", "serde_json", "sqlx", @@ -16081,7 +16779,7 @@ dependencies = [ [[package]] name = "windmill-parser-r" -version = "1.821.0" +version = "1.830.0" dependencies = [ "anyhow", "serde_json", @@ -16093,7 +16791,7 @@ dependencies = [ [[package]] name = "windmill-parser-ruby" -version = "1.821.0" +version = "1.830.0" dependencies = [ "anyhow", "lazy_static", @@ -16107,7 +16805,7 @@ dependencies = [ [[package]] name = "windmill-parser-rust" -version = "1.821.0" +version = "1.830.0" dependencies = [ "anyhow", "convert_case 0.6.0", @@ -16124,7 +16822,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql" -version = "1.821.0" +version = "1.830.0" dependencies = [ "anyhow", "lazy_static", @@ -16137,7 +16835,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql-asset" -version = "1.821.0" +version = "1.830.0" dependencies = [ "anyhow", "serde", @@ -16149,7 +16847,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts" -version = "1.821.0" +version = "1.830.0" dependencies = [ "anyhow", "lazy_static", @@ -16167,7 +16865,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts-asset" -version = "1.821.0" +version = "1.830.0" dependencies = [ "anyhow", "serde-wasm-bindgen", @@ -16183,11 +16881,12 @@ dependencies = [ [[package]] name = "windmill-parser-wac" -version = "1.821.0" +version = "1.830.0" dependencies = [ "anyhow", - "rustpython-ast", - "rustpython-parser", + "ruff_python_ast", + "ruff_python_parser", + "ruff_text_size", "serde", "serde_json", "sha2 0.10.9", @@ -16199,7 +16898,7 @@ dependencies = [ [[package]] name = "windmill-parser-yaml" -version = "1.821.0" +version = "1.830.0" dependencies = [ "anyhow", "lazy_static", @@ -16213,11 +16912,11 @@ dependencies = [ [[package]] name = "windmill-queue" -version = "1.821.0" +version = "1.830.0" dependencies = [ "anyhow", "async-recursion", - "axum 0.8.9", + "axum", "backon", "chrono", "chrono-tz", @@ -16226,7 +16925,7 @@ dependencies = [ "futures", "futures-core", "hex", - "hmac", + "hmac 0.12.1", "itertools 0.14.0", "lazy_static", "once_cell", @@ -16252,23 +16951,19 @@ dependencies = [ [[package]] name = "windmill-runtime-nativets" -version = "1.821.0" +version = "1.830.0" dependencies = [ "anyhow", "const_format", "deno_ast", - "deno_console", "deno_core", "deno_crypto", - "deno_error 0.6.1", + "deno_error", "deno_fetch", - "deno_fs", - "deno_io", "deno_net", "deno_permissions", "deno_telemetry", "deno_tls", - "deno_url", "deno_web", "deno_webidl", "futures", @@ -16277,13 +16972,15 @@ dependencies = [ "rcgen", "regex", "reqwest 0.13.5", - "rustls 0.23.35", + "rustls 0.23.40", "serde", "serde_json", "sqlx", + "sys_traits", "tokio", "tracing", "uuid", + "v8", "winapi", "windmill-common", "windmill-parser-ts", @@ -16292,7 +16989,7 @@ dependencies = [ [[package]] name = "windmill-sql-datatype-parser-wasm" -version = "1.821.0" +version = "1.830.0" dependencies = [ "getrandom 0.3.4", "wasm-bindgen", @@ -16303,18 +17000,18 @@ dependencies = [ [[package]] name = "windmill-store" -version = "1.821.0" +version = "1.830.0" dependencies = [ "anyhow", "async-recursion", - "axum 0.8.9", + "axum", "base64 0.22.1", "chrono", "dashmap", "futures", "hex", "http 1.5.0", - "hyper 1.11.1", + "hyper 1.12.0", "lazy_static", "magic-crypt", "quick_cache", @@ -16338,11 +17035,11 @@ dependencies = [ [[package]] name = "windmill-test-utils" -version = "1.821.0" +version = "1.830.0" dependencies = [ "anyhow", "async-trait", - "axum 0.8.9", + "axum", "chrono", "futures", "serde", @@ -16362,14 +17059,14 @@ dependencies = [ [[package]] name = "windmill-trigger" -version = "1.821.0" +version = "1.830.0" dependencies = [ "anyhow", "async-trait", - "axum 0.8.9", + "axum", "chrono", "http 1.5.0", - "hyper 1.11.1", + "hyper 1.12.0", "itertools 0.14.0", "lazy_static", "rand 0.9.5", @@ -16395,11 +17092,11 @@ dependencies = [ [[package]] name = "windmill-trigger-amqp" -version = "1.821.0" +version = "1.830.0" dependencies = [ "anyhow", "async-trait", - "axum 0.8.9", + "axum", "base64 0.22.1", "futures", "itertools 0.14.0", @@ -16422,11 +17119,11 @@ dependencies = [ [[package]] name = "windmill-trigger-azure" -version = "1.821.0" +version = "1.830.0" dependencies = [ "anyhow", "async-trait", - "axum 0.8.9", + "axum", "base64 0.22.1", "bytes", "chrono", @@ -16455,11 +17152,11 @@ dependencies = [ [[package]] name = "windmill-trigger-email" -version = "1.821.0" +version = "1.830.0" dependencies = [ "anyhow", "async-trait", - "axum 0.8.9", + "axum", "base64 0.22.1", "lazy_static", "regex", @@ -16475,11 +17172,11 @@ dependencies = [ [[package]] name = "windmill-trigger-gcp" -version = "1.821.0" +version = "1.830.0" dependencies = [ "anyhow", "async-trait", - "axum 0.8.9", + "axum", "base64 0.22.1", "bytes", "chrono", @@ -16509,19 +17206,19 @@ dependencies = [ [[package]] name = "windmill-trigger-http" -version = "1.821.0" +version = "1.830.0" dependencies = [ "anyhow", "async-trait", - "axum 0.8.9", + "axum", "base64 0.22.1", "chrono", "constant_time_eq 0.3.1", "futures", "hex", - "hmac", + "hmac 0.12.1", "http 1.5.0", - "hyper 1.11.1", + "hyper 1.12.0", "itertools 0.14.0", "lazy_static", "matchit 0.7.3", @@ -16546,11 +17243,11 @@ dependencies = [ [[package]] name = "windmill-trigger-kafka" -version = "1.821.0" +version = "1.830.0" dependencies = [ "anyhow", "async-trait", - "axum 0.8.9", + "axum", "base64 0.22.1", "itertools 0.14.0", "rdkafka", @@ -16569,11 +17266,11 @@ dependencies = [ [[package]] name = "windmill-trigger-mqtt" -version = "1.821.0" +version = "1.830.0" dependencies = [ "anyhow", "async-trait", - "axum 0.8.9", + "axum", "base64 0.22.1", "bytes", "itertools 0.14.0", @@ -16593,12 +17290,12 @@ dependencies = [ [[package]] name = "windmill-trigger-nats" -version = "1.821.0" +version = "1.830.0" dependencies = [ "anyhow", "async-nats", "async-trait", - "axum 0.8.9", + "axum", "base64 0.22.1", "itertools 0.14.0", "nkeys", @@ -16617,11 +17314,11 @@ dependencies = [ [[package]] name = "windmill-trigger-postgres" -version = "1.821.0" +version = "1.830.0" dependencies = [ "anyhow", "async-trait", - "axum 0.8.9", + "axum", "byteorder", "bytes", "chrono", @@ -16652,7 +17349,7 @@ dependencies = [ [[package]] name = "windmill-trigger-sqs" -version = "1.821.0" +version = "1.830.0" dependencies = [ "anyhow", "async-trait", @@ -16661,7 +17358,7 @@ dependencies = [ "aws-sdk-sqs", "aws-sdk-sts", "aws-smithy-types", - "axum 0.8.9", + "axum", "backon", "chrono", "itertools 0.14.0", @@ -16680,11 +17377,11 @@ dependencies = [ [[package]] name = "windmill-trigger-websocket" -version = "1.821.0" +version = "1.830.0" dependencies = [ "anyhow", "async-trait", - "axum 0.8.9", + "axum", "base64 0.22.1", "futures", "http 1.5.0", @@ -16705,7 +17402,7 @@ dependencies = [ [[package]] name = "windmill-types" -version = "1.821.0" +version = "1.830.0" dependencies = [ "anyhow", "bitflags 2.13.2", @@ -16724,14 +17421,14 @@ dependencies = [ [[package]] name = "windmill-worker" -version = "1.821.0" +version = "1.830.0" dependencies = [ "anyhow", "async-once-cell", "async-recursion", "async-stream", "async-trait", - "axum 0.8.9", + "axum", "backon", "base64 0.22.1", "bit-vec 0.6.3", @@ -16750,7 +17447,7 @@ dependencies = [ "gcp_auth", "git-version", "hex", - "hmac", + "hmac 0.12.1", "hudsucker", "hyper-http-proxy", "hyper-rustls 0.27.10", @@ -16773,9 +17470,10 @@ dependencies = [ "pem", "pep440_rs", "postgres-native-tls 0.5.3", + "postgres-protocol", "process-wrap", "prometheus", - "prost", + "prost 0.13.5", "quick_cache", "rand 0.9.5", "rcgen", @@ -16784,7 +17482,7 @@ dependencies = [ "reqwest-middleware", "rsa", "rust_decimal", - "rustls 0.23.35", + "rustls 0.23.40", "rustls-native-certs 0.8.4", "rustls-pemfile 2.2.0", "serde", @@ -16842,7 +17540,7 @@ dependencies = [ [[package]] name = "windmill-worker-volumes" -version = "1.821.0" +version = "1.830.0" dependencies = [ "bytes", "futures", @@ -17437,9 +18135,26 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1301e935010a701ae5f8655edc0ad17c44bad3ac5ce8c39185f75453b720ae94" dependencies = [ - "const-oid", - "der", - "spki", + "const-oid 0.9.6", + "der 0.7.10", + "spki 0.7.3", +] + +[[package]] +name = "x509-parser" +version = "0.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7069fba5b66b9193bd2c5d3d4ff12b839118f6bcbef5328efafafb5395cf63da" +dependencies = [ + "asn1-rs 0.5.2", + "data-encoding", + "der-parser 8.2.0", + "lazy_static", + "nom", + "oid-registry 0.6.1", + "rusticata-macros", + "thiserror 1.0.69", + "time", ] [[package]] @@ -17551,18 +18266,18 @@ dependencies = [ [[package]] name = "zerocopy" -version = "0.8.59" +version = "0.8.62" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6df92bf3d9227be3d53173901ddbffac2babc27ae50f397776ffd6dc33f800cb" +checksum = "86502bf56ac7c77571a32e2647bb2a15894565e981fb2a48d7bde2d91c965a9d" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.59" +version = "0.8.62" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac4f328cf2f05d084e496c3e9c3f33ed0a183656a16e1fcec4d464d8373aec82" +checksum = "5457206954b06561e2608c7e19cf58b1926586d999c246eebe4502f7e2039d1a" dependencies = [ "proc-macro2", "quote", @@ -17592,9 +18307,9 @@ dependencies = [ [[package]] name = "zeroize" -version = "1.9.0" +version = "1.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" +checksum = "e13084392c5e4bc371903e2935a5eaeed24905a7511356b883835e18a78f6879" dependencies = [ "zeroize_derive", ] @@ -17669,6 +18384,19 @@ version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" +[[package]] +name = "zoneinfo64" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed6eb2607e906160c457fd573e9297e65029669906b9ac8fb1b5cd5e055f0705" +dependencies = [ + "calendrical_calculations", + "icu_locale_core", + "potential_utf", + "resb", + "serde", +] + [[package]] name = "zstd" version = "0.13.3" @@ -17689,9 +18417,9 @@ dependencies = [ [[package]] name = "zstd-sys" -version = "2.1.0+zstd.1.5.7" +version = "2.1.1+zstd.1.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ef0a8027ec3ee71300ab3bcbcd0393f434aa72b91ca6d635a39941deae8eea0" +checksum = "aeec9eaf2dffbbd09201e23bd0ffcbaa33bb8e9266a10734fd7ed90a85eca078" dependencies = [ "cc", "pkg-config", diff --git a/backend/Cargo.toml b/backend/Cargo.toml index d2bec22817..aaf5a2e8df 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "windmill" -version = "1.821.0" +version = "1.830.0" authors.workspace = true edition.workspace = true @@ -88,7 +88,7 @@ members = [ exclude = ["./windmill-duckdb-ffi-internal", "./parsers/windmill-parser-wasm"] [workspace.package] -version = "1.821.0" +version = "1.830.0" authors = ["Ruben Fiszel "] edition = "2021" @@ -156,7 +156,7 @@ bedrock = ["windmill-ai/bedrock", "windmill-api/bedrock", "windmill-worker/bedro mqtt_trigger = ["windmill-api/mqtt_trigger"] amqp_trigger = ["windmill-api/amqp_trigger"] native_trigger = ["windmill-api/native_trigger"] -sqs_trigger = ["windmill-api/sqs_trigger", "windmill-common/aws_auth", "windmill-api/openidconnect"] +sqs_trigger = ["windmill-api/sqs_trigger", "windmill-common/aws_auth", "openidconnect"] gcp_trigger = ["windmill-api/gcp_trigger"] azure_trigger = ["windmill-api/azure_trigger"] smtp = ["windmill-api/smtp", "windmill-common/smtp", "windmill-queue/smtp"] @@ -249,36 +249,22 @@ sqlx-sqlite = { git = "https://github.com/windmill-labs/sqlx", rev = "6bdaee94fa object_store = { git = "https://github.com/apache/arrow-rs-object-store", rev = "36752c975d4f29e20b57c91f81a10872dcd48ae7" } # Use tiberius main branch for libgssapi 0.8.1 fix (https://github.com/prisma/tiberius/issues/343) tiberius = { git = "https://github.com/prisma/tiberius", rev = "59db57960a14b422fb3a1309aa4aa47880896ff8" } -# Pin tokio-postgres / postgres-types / postgres-protocol to the -# MaterializeInc fork. windmill-trigger-postgres already pulled this -# fork in transitively for the postgres-replication crate -# (CopyBothDuplex, LogicalReplicationStream, TupleData with binary -# tuple support) which upstream rust-postgres has declined to merge -# since 2021 (PR #752 → #778, both still unmerged). +# Pin tokio-postgres / postgres-types / postgres-protocol to +# windmill-labs/rust-postgres, branch `windmill-describe-first`: the +# MaterializeInc fork (postgres-replication, which upstream has not merged) +# plus `Client::describe_typed`, which pg_executor needs before +# query_typed_raw so result columns of types the client has not seen yet +# (custom enums / domains, citext, postgis) don't deadlock a large result, +# and `Client::transaction_status`, which keeps a connection left inside a +# transaction out of the executor's cache. Each pinned commit carries a `windmill-*` tag so it stays reachable when the +# branch is rebased onto MaterializeInc master. # -# MI also carries a mitigation for the -# Client::query_typed_raw / Client::prepare deadlock on result columns -# whose Oid the client doesn't know about yet (citext, custom enums / -# domains, postgis): MI's 2025-12-11 PR #33 resized the per-request -# response channel from mpsc::channel(1) → mpsc::channel(1024). -# bounded(1024) is sufficient for any realistic typeinfo deferral -# (need ~2-3 batches) but leaves a theoretical failure mode at -# >~64 MB results with a custom-Oid column. The strict-correct fix is -# mpsc::unbounded(); a follow-up PR to MI is open proposing that. -# -# The [patch.crates-io] entries below force windmill-worker's -# pg_executor (which imports `tokio_postgres::` directly from -# crates.io) onto the same fork as windmill-trigger-postgres, so the -# deadlock mitigation reaches both consumers. -# -# Upstream deadlock PRs (open, not on the critical path now that MI -# is mitigated): -# https://github.com/rust-postgres/rust-postgres/pull/1348 -# https://github.com/rust-postgres/rust-postgres/pull/1349 -# Reproducer: https://github.com/rubenfiszel/tokio-postgres-deadlock-repro -tokio-postgres = { git = "https://github.com/MaterializeInc/rust-postgres", rev = "78c1222577bb091d69bc22b1bc7ad01c14675abe" } -postgres-types = { git = "https://github.com/MaterializeInc/rust-postgres", rev = "78c1222577bb091d69bc22b1bc7ad01c14675abe" } -postgres-protocol = { git = "https://github.com/MaterializeInc/rust-postgres", rev = "78c1222577bb091d69bc22b1bc7ad01c14675abe" } +# The [patch.crates-io] entries force windmill-worker's pg_executor (which +# imports `tokio_postgres::` from crates.io) onto the same fork as +# windmill-trigger-postgres. +tokio-postgres = { git = "https://github.com/windmill-labs/rust-postgres", rev = "b3a6443aa1ae90a4f6c4b742103ba80584598f2e" } +postgres-types = { git = "https://github.com/windmill-labs/rust-postgres", rev = "b3a6443aa1ae90a4f6c4b742103ba80584598f2e" } +postgres-protocol = { git = "https://github.com/windmill-labs/rust-postgres", rev = "b3a6443aa1ae90a4f6c4b742103ba80584598f2e" } [dependencies] anyhow.workspace = true @@ -295,6 +281,7 @@ windmill-api = { workspace = true, default-features = false } windmill-api-agent-workers = { workspace = true, optional = true } windmill-api-scripts.workspace = true windmill-api-settings.workspace = true +windmill-audit.workspace = true windmill-worker.workspace = true windmill-indexer = { workspace = true, optional = true } windmill-autoscaling = { workspace = true, optional = true } @@ -465,7 +452,7 @@ headers = "^0" hyper = { version = "^1", features = ["full"] } hyper-tls = "^0.6" hyper-util = { version = "^0.1", features = ["client-legacy", "http1", "tokio"] } -tokio = { version = "=1.46.1", features = ["full", "tracing", "time"] } +tokio = { version = "=1.53.2", features = ["full", "tracing", "time"] } tokio-stream = { version = "0.1.17" } tower = "^0" tower-http = { version = "^0.6", features = ["trace", "cors", "catch-panic"] } @@ -473,7 +460,7 @@ tower-cookies = "^0.11" serde = "^1" # 1.0.151 introduced RawValue::from_string_unchecked, which the SQL executors use # to avoid re-parsing every collected row. -serde_json = { version = "^1.0.151", features = ["preserve_order", "raw_value"] } +serde_json = { version = "^1.0.151", features = ["preserve_order", "raw_value", "float_roundtrip"] } serde_yml = "0.0.12" uuid = { version = "^1", features = ["serde", "v4", "js"] } thiserror = "^2" @@ -502,9 +489,11 @@ arc-swap = "1" ed25519-dalek = { version = "2", features = ["rand_core"] } magic-crypt = "^3" git-version = "^0" -malachite = "=0.4.18" -malachite-bigint = "=0.2.0" -rustpython-parser = "^0" +# Ruff's parser crates are internal to ruff and carry no semver guarantee: pin them exactly, +# and to the same release. +ruff_python_parser = "=0.0.17" +ruff_python_ast = "=0.0.17" +ruff_text_size = "=0.0.17" pep440_rs = "0.7.3" php-parser-rs = { git = "https://github.com/php-rust-tools/parser", rev = "ec4cb411dec09450946ef57920b7ffced7f6495d" } cron = "^0" @@ -533,33 +522,29 @@ aws-sdk-rds = "^1" async-trait = "0.1.88" -v8 = "=137.1.0" # Exact version NOTE: Do not forget to update version and hash in flake.nix -# deno_* pin set: deno v2.4.0 base, with deno_ast force-overridden to =0.51.0. -# Rationale: deno_ast 0.51.0 is the first version pulling swc_common =14.0.4, -# the first swc_common patch that dropped `pub use serde::__private as serde;` -# (the line that capped our workspace serde pin at =1.0.220). v2.4.0's other -# pins keep deno_tls at 0.196.0 which uses permissive `rustls ^0.23.11`, -# compatible with aws-sdk-bedrockruntime's `^0.23.31` requirement. deno_tls -# 0.198+ tightened that to exact `=0.23.28`, which would have made any -# meaningful deno bump resolver-impossible against aws-sdk. -deno_fetch = "0.233.0" -deno_tls = "0.196.0" -deno_console = "0.209.0" -deno_url = "0.209.0" -deno_webidl = "0.209.0" -deno_web = "0.240.0" -# Sibling release of the pinned deno_core 0.352 / deno_web 0.240 stack -# (its deps are deno_core ^0.352, deno_web ^0.240, deno_error =0.6.1). A newer -# deno_crypto would force bumping the whole deno stack. -deno_crypto = "0.223.0" -deno_io = "0.119.0" -deno_fs = "0.119.0" -deno_net = "0.201.0" -deno_core = "0.352.0" +# deno_* pin set: the crates deno published together on 2026-09-16 (deno_core +# 0.412). +# V8 is pinned exactly because flake.nix fetches the prebuilt archive of that +# version: update its version and hash there with any bump. No default features, +# so the pin leaves the feature set deno_core picks untouched. +v8 = { version = "=150.4.0", default-features = false } +# These release as a matched set: each pins its siblings, deno_error exactly, +# and deno_tls pins rustls exactly (=0.23.40, the workspace `rustls` pin below +# must equal it). A bump moves all of them at once. +# deno_ast is independent of that set and stays at 0.51.0: a newer one pulls +# swc_common 17, which would force bumping every swc_* pin the parsers use. +deno_fetch = "0.283.0" +deno_tls = "0.246.0" +deno_webidl = "0.259.0" +deno_web = "0.290.0" +deno_crypto = "0.273.0" +deno_net = "0.251.0" +deno_core = "0.412.0" deno_ast = { version = "=0.51.0", features = ["transpiling"] } -deno_permissions = "0.68.0" -deno_telemetry = "0.31.0" -deno_error = "=0.6.1" +deno_permissions = "0.118.0" +deno_telemetry = "0.81.0" +deno_error = "=0.7.1" +sys_traits = { version = "0.1.28", features = ["real", "libc", "winapi"] } rustls-pemfile = "2.2.0" # only used with special deno_core_mac feature to prevent ffi issue on macos, requires libffi to be installed @@ -586,6 +571,7 @@ base64 = "^0.22.1" base32 = "^0" hmac = "0.12.1" sha2 = "0.10.6" +ryu-js = "1.0.3" md-5 = "0.10.6" sha1 = "0.10.6" sqlx = { version = "0.8.0", features = [ @@ -609,9 +595,11 @@ const_format = { version = "0.2.35", features = ["rust_1_64", "rust_1_51"] } const-str = "0.5" constant_time_eq = "0.3.1" rsa = "^0" +p256 = "0.13" spki = { version = "0.7", features = ["pem"] } pkcs1 = "0.7" aes-gcm = "0.10.3" +hkdf = "0.12.4" async_zip = { version = "0.0.17", features = ["tokio", "tokio-fs", "deflate", "chrono"] } once_cell = "1.17.1" dashmap = "6.1.0" @@ -630,8 +618,8 @@ convert_case = "0.6.0" getrandom = "0.2" tokio-postgres = {version = "^0.7", features = ["array-impls", "with-serde_json-1", "with-chrono-0_4", "with-uuid-1", "with-bit-vec-0_6"]} postgres-protocol = "0.6" -rust-postgres = { package = "tokio-postgres", git = "https://github.com/MaterializeInc/rust-postgres", rev = "78c1222577bb091d69bc22b1bc7ad01c14675abe"} -rust-postgres-native-tls = { package = "postgres-native-tls", git = "https://github.com/MaterializeInc/rust-postgres", features = ["runtime"], rev = "78c1222577bb091d69bc22b1bc7ad01c14675abe" } +rust-postgres = { package = "tokio-postgres", git = "https://github.com/windmill-labs/rust-postgres", rev = "b3a6443aa1ae90a4f6c4b742103ba80584598f2e"} +rust-postgres-native-tls = { package = "postgres-native-tls", git = "https://github.com/windmill-labs/rust-postgres", features = ["runtime"], rev = "b3a6443aa1ae90a4f6c4b742103ba80584598f2e" } bit-vec = "=0.6.3" mappable-rc = "^0" mysql_async = { version = "*", default-features = false, features = ["minimal", "default", "native-tls-tls", "rust_decimal"]} @@ -695,7 +683,7 @@ aws-sdk-sqs = "^1.89.0" aws-sdk-sts = "^1.91.0" aws-sdk-sso = "=1.77.0" aws-sdk-ssooidc = "=1.78.0" -rustls = "=0.23.35" +rustls = "=0.23.40" async-once-cell = "0.5.4" size = "0.5.0" diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index c2c253e8f0..695f6d248b 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -ff4d04f17721d84fb9a1f655af50ff7e4e7dcbc7 +40ab35596e6fcb82a4dfc56e186befb807f8e2af diff --git a/backend/migrations/20260929064555_flow_conversation_message_turn_job_id.down.sql b/backend/migrations/20260929064555_flow_conversation_message_turn_job_id.down.sql new file mode 100644 index 0000000000..5906eda25d --- /dev/null +++ b/backend/migrations/20260929064555_flow_conversation_message_turn_job_id.down.sql @@ -0,0 +1 @@ +ALTER TABLE flow_conversation_message DROP COLUMN turn_job_id; diff --git a/backend/migrations/20260929064555_flow_conversation_message_turn_job_id.up.sql b/backend/migrations/20260929064555_flow_conversation_message_turn_job_id.up.sql new file mode 100644 index 0000000000..db9427692b --- /dev/null +++ b/backend/migrations/20260929064555_flow_conversation_message_turn_job_id.up.sql @@ -0,0 +1,23 @@ +-- The flow run a row's turn belongs to: the job the user message started. Retention collects +-- rows by it, so a turn goes as one unit. `job_id` names the step or tool job that wrote the +-- row, which completes, and so expires, before the run does. +ALTER TABLE flow_conversation_message ADD COLUMN turn_job_id UUID; + +-- Turns can overlap in one conversation, so a row's turn is the user row its job descends +-- from. Transcript order only decides for rows whose job is already gone or that have none. +UPDATE flow_conversation_message m + SET turn_job_id = COALESCE( + (SELECT u.job_id FROM v2_job j + JOIN flow_conversation_message u + ON u.conversation_id = m.conversation_id + AND u.message_type = 'user' + AND u.job_id IN (j.id, j.parent_job, j.root_job, j.flow_innermost_root_job) + WHERE j.id = m.job_id + LIMIT 1), + (SELECT u.job_id FROM flow_conversation_message u + WHERE u.conversation_id = m.conversation_id + AND u.message_type = 'user' + AND u.created_seq <= m.created_seq + ORDER BY u.created_seq DESC + LIMIT 1) + ); diff --git a/backend/migrations/20261002124918_job_token_scopes.down.sql b/backend/migrations/20261002124918_job_token_scopes.down.sql new file mode 100644 index 0000000000..be2c13b3ff --- /dev/null +++ b/backend/migrations/20261002124918_job_token_scopes.down.sql @@ -0,0 +1,3 @@ +ALTER TABLE job_perms DROP COLUMN IF EXISTS job_token_scopes; +ALTER TABLE flow DROP COLUMN IF EXISTS job_token_scopes; +ALTER TABLE script DROP COLUMN IF EXISTS job_token_scopes; diff --git a/backend/migrations/20261002124918_job_token_scopes.up.sql b/backend/migrations/20261002124918_job_token_scopes.up.sql new file mode 100644 index 0000000000..6b24325378 --- /dev/null +++ b/backend/migrations/20261002124918_job_token_scopes.up.sql @@ -0,0 +1,3 @@ +ALTER TABLE script ADD COLUMN IF NOT EXISTS job_token_scopes text[]; +ALTER TABLE flow ADD COLUMN IF NOT EXISTS job_token_scopes text[]; +ALTER TABLE job_perms ADD COLUMN IF NOT EXISTS job_token_scopes text[]; diff --git a/backend/migrations/20261008070335_worker_ping_draining.down.sql b/backend/migrations/20261008070335_worker_ping_draining.down.sql new file mode 100644 index 0000000000..64a5f2949d --- /dev/null +++ b/backend/migrations/20261008070335_worker_ping_draining.down.sql @@ -0,0 +1 @@ +ALTER TABLE worker_ping DROP COLUMN IF EXISTS draining; diff --git a/backend/migrations/20261008070335_worker_ping_draining.up.sql b/backend/migrations/20261008070335_worker_ping_draining.up.sql new file mode 100644 index 0000000000..33b5ad5e3e --- /dev/null +++ b/backend/migrations/20261008070335_worker_ping_draining.up.sql @@ -0,0 +1 @@ +ALTER TABLE worker_ping ADD COLUMN IF NOT EXISTS draining BOOLEAN NOT NULL DEFAULT FALSE; diff --git a/backend/migrations/20261008165042_runnable_job_stats.down.sql b/backend/migrations/20261008165042_runnable_job_stats.down.sql new file mode 100644 index 0000000000..21208736b4 --- /dev/null +++ b/backend/migrations/20261008165042_runnable_job_stats.down.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS runnable_job_stats; diff --git a/backend/migrations/20261008165042_runnable_job_stats.up.sql b/backend/migrations/20261008165042_runnable_job_stats.up.sql new file mode 100644 index 0000000000..cf676a037e --- /dev/null +++ b/backend/migrations/20261008165042_runnable_job_stats.up.sql @@ -0,0 +1,17 @@ +-- Hourly resource rollup per runnable and worker group, fed by the workers. +-- Memory figures are in kB, like v2_job_completed.memory_peak. +-- No foreign key to workspace: every worker upserts here, and the key would cost each row +-- a lookup and a lock on its workspace. Workspace deletion removes the rows itself. +CREATE TABLE IF NOT EXISTS runnable_job_stats ( + hour BIGINT NOT NULL, + workspace_id VARCHAR(50) NOT NULL, + runnable_path VARCHAR(255) NOT NULL, + worker_group TEXT NOT NULL, + job_count INTEGER NOT NULL DEFAULT 0, + total_duration_ms BIGINT NOT NULL DEFAULT 0, + max_memory_peak INTEGER NOT NULL DEFAULT 0, + sum_memory_peak BIGINT NOT NULL DEFAULT 0, + memory_sample_count INTEGER NOT NULL DEFAULT 0, + total_cpu_ms BIGINT NOT NULL DEFAULT 0, + PRIMARY KEY (hour, workspace_id, runnable_path, worker_group) +); diff --git a/backend/migrations/20261012180101_datatable_role_set.down.sql b/backend/migrations/20261012180101_datatable_role_set.down.sql new file mode 100644 index 0000000000..15384edae8 --- /dev/null +++ b/backend/migrations/20261012180101_datatable_role_set.down.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS datatable_role_set; diff --git a/backend/migrations/20261012180101_datatable_role_set.up.sql b/backend/migrations/20261012180101_datatable_role_set.up.sql new file mode 100644 index 0000000000..0093837d3b --- /dev/null +++ b/backend/migrations/20261012180101_datatable_role_set.up.sql @@ -0,0 +1,21 @@ +-- The logins behind a data table connection made as several roles at once: one Postgres login per +-- distinct set of data table roles, a member of each. Which roles a login holds is its +-- memberships on the cluster, and its name is derived from their catalog ids, so a row only has to +-- carry the credential. +-- +-- A row is written after its login exists and removed before the login is dropped, so a row always +-- names a login that can authenticate with `pwd`. +CREATE TABLE datatable_role_set ( + cluster VARCHAR(20) NOT NULL CHECK (cluster IN ('instance', 'external_instance')), + name VARCHAR(63) NOT NULL, + -- Generated by Windmill, never entered by anyone, and never leaves the server. + pwd TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + -- Recorded at most once a day. A login unused for long enough is dropped, and made again if + -- its set of roles is asked for later. + last_used_at TIMESTAMPTZ NOT NULL DEFAULT now(), + PRIMARY KEY (cluster, name) +); + +GRANT ALL ON datatable_role_set TO windmill_user; +GRANT ALL ON datatable_role_set TO windmill_admin; diff --git a/backend/migrations/20261012213357_widen_variable_value.down.sql b/backend/migrations/20261012213357_widen_variable_value.down.sql new file mode 100644 index 0000000000..48b61c3f65 --- /dev/null +++ b/backend/migrations/20261012213357_widen_variable_value.down.sql @@ -0,0 +1,12 @@ +DO $$ +DECLARE + too_long BIGINT; +BEGIN + SELECT count(*) INTO too_long FROM variable WHERE length(value) > 15000; + IF too_long > 0 THEN + RAISE EXCEPTION 'cannot narrow variable.value to VARCHAR(15000): % row(s) are longer than 15000 characters', too_long; + END IF; +END $$; + +ALTER TABLE variable DROP CONSTRAINT IF EXISTS variable_value_length; +ALTER TABLE variable ALTER COLUMN value TYPE VARCHAR(15000); diff --git a/backend/migrations/20261012213357_widen_variable_value.up.sql b/backend/migrations/20261012213357_widen_variable_value.up.sql new file mode 100644 index 0000000000..06dc8a191a --- /dev/null +++ b/backend/migrations/20261012213357_widen_variable_value.up.sql @@ -0,0 +1,8 @@ +-- AES-256-GCM ciphertext is 32 bytes (magic, nonce, tag) longer than legacy CBC before base64, +-- so the longest CBC value that fits 15000 chars is exactly 15040 chars once re-encrypted. +ALTER TABLE variable ALTER COLUMN value TYPE VARCHAR(15040); + +-- Only GCM secrets (base64 of the `wmA2` magic starts `d21BM`) may use the extra room: a longer +-- CBC value could never be re-encrypted to fit, and would fail every rotation of its workspace key. +ALTER TABLE variable ADD CONSTRAINT variable_value_length + CHECK (length(value) <= 15000 OR (is_secret AND value LIKE 'd21BM%')); diff --git a/backend/oauth_connect.json b/backend/oauth_connect.json index 2f889336f9..f7c7ef9698 100644 --- a/backend/oauth_connect.json +++ b/backend/oauth_connect.json @@ -45,12 +45,39 @@ "files:read" ], "scope_options": [ + { "value": "chat:write", "label": "Send messages as the app" }, + { "value": "chat:write.public", "label": "Send messages to public channels the app is not in" }, + { "value": "channels:join", "label": "Join public channels" }, + { "value": "files:write", "label": "Upload, edit and delete files" }, + { "value": "users:read", "label": "View people in the workspace" }, + { "value": "users:read.email", "label": "View people's email addresses" }, + { "value": "channels:read", "label": "View basic information about public channels" }, + { "value": "channels:history", "label": "Read messages in public channels the app is in" }, { "value": "reactions:write", "label": "Add emoji reactions" }, { "value": "channels:manage", "label": "Create, archive and manage public channels, invite members, set topics" }, { "value": "groups:write", "label": "Create and manage private channels the app is in" }, - { "value": "usergroups:write", "label": "Edit user groups (workspace settings often block bots)" } + { "value": "usergroups:write", "label": "Edit user groups (workspace settings often block bots)" }, + { "value": "files:read", "label": "Read and download files shared where the app is a member" }, + { "value": "groups:read", "label": "List private channels the app is in" }, + { "value": "im:read", "label": "List the app's direct messages" }, + { "value": "mpim:read", "label": "List the app's group direct messages" }, + { "value": "usergroups:read", "label": "List user groups" }, + { "value": "emoji:read", "label": "List custom emojis" } ], "user_scope_options": [ + { "value": "channels:history", "label": "Read messages in public channels" }, + { "value": "groups:history", "label": "Read messages in your private channels" }, + { "value": "im:history", "label": "Read your direct messages" }, + { "value": "mpim:history", "label": "Read your group direct messages" }, + { "value": "search:read", "label": "Search messages and files" }, + { "value": "users:read", "label": "View people in the workspace" }, + { "value": "chat:write", "label": "Send messages as you" }, + { "value": "users:read.email", "label": "View people's email addresses" }, + { "value": "channels:read", "label": "View basic information about public channels" }, + { "value": "groups:read", "label": "View basic information about your private channels" }, + { "value": "im:read", "label": "View basic information about your direct messages" }, + { "value": "mpim:read", "label": "View basic information about your group direct messages" }, + { "value": "files:read", "label": "View files shared in your conversations" }, { "value": "reactions:write", "label": "Add emoji reactions" }, { "value": "channels:write", "label": "Create, archive and manage public channels" }, { "value": "channels:write.invites", "label": "Invite members to public channels" }, @@ -58,7 +85,10 @@ { "value": "groups:write", "label": "Create and manage private channels" }, { "value": "users.profile:write", "label": "Set your status and edit your profile" }, { "value": "reminders:write", "label": "Create reminders for yourself" }, - { "value": "usergroups:write", "label": "Edit user groups" } + { "value": "usergroups:write", "label": "Edit user groups" }, + { "value": "files:write", "label": "Upload and delete files as yourself" }, + { "value": "usergroups:read", "label": "List user groups" }, + { "value": "emoji:read", "label": "List custom emojis" } ], "token_response_path": "authed_user" }, @@ -73,7 +103,8 @@ "scopes": ["https://www.googleapis.com/auth/spreadsheets"], "scope_options": [ { "value": "https://www.googleapis.com/auth/spreadsheets", "label": "Read, edit, create and delete spreadsheets" }, - { "value": "https://www.googleapis.com/auth/spreadsheets.readonly", "label": "Read spreadsheets" } + { "value": "https://www.googleapis.com/auth/spreadsheets.readonly", "label": "Read spreadsheets" }, + { "value": "https://www.googleapis.com/auth/drive.metadata.readonly", "label": "List and search your spreadsheets in Drive" } ], "extra_params": { "access_type": "offline", @@ -150,7 +181,9 @@ "scope_options": [ { "value": "https://www.googleapis.com/auth/documents", "label": "Read, edit, create and delete documents" }, { "value": "https://www.googleapis.com/auth/documents.readonly", "label": "Read documents" }, - { "value": "https://www.googleapis.com/auth/drive.file", "label": "Only files created or opened with this connection" } + { "value": "https://www.googleapis.com/auth/drive.file", "label": "Only files created or opened with this connection" }, + { "value": "https://www.googleapis.com/auth/drive.readonly", "label": "Read, export and search all Drive files and comments" }, + { "value": "https://www.googleapis.com/auth/drive", "label": "Read, edit, create and delete all Drive files" } ], "extra_params": { "access_type": "offline", @@ -205,7 +238,8 @@ { "value": "https://www.googleapis.com/auth/admin.directory.group.readonly", "label": "Read groups" }, { "value": "https://www.googleapis.com/auth/admin.directory.orgunit", "label": "Read and manage organizational units" }, { "value": "https://www.googleapis.com/auth/admin.directory.orgunit.readonly", "label": "Read organizational units" }, - { "value": "https://www.googleapis.com/auth/admin.directory.user.security", "label": "Manage user security settings" } + { "value": "https://www.googleapis.com/auth/admin.directory.user.security", "label": "Manage user security settings" }, + { "value": "https://www.googleapis.com/auth/admin.reports.audit.readonly", "label": "Read audit logs (login, admin, Drive, token activity)" } ], "extra_params": { "access_type": "offline", @@ -220,6 +254,14 @@ "type": "web_server" } }, + "notion": { + "auth_url": "https://api.notion.com/v1/oauth/authorize", + "token_url": "https://api.notion.com/v1/oauth/token", + "scopes": [], + "extra_params": { + "owner": "user" + } + }, "linkedin": { "auth_url": "https://www.linkedin.com/oauth/v2/authorization", "token_url": "https://www.linkedin.com/oauth/v2/accessToken", diff --git a/backend/parsers/windmill-parser-py-asset/Cargo.toml b/backend/parsers/windmill-parser-py-asset/Cargo.toml index ddd6446079..84562dcf0e 100644 --- a/backend/parsers/windmill-parser-py-asset/Cargo.toml +++ b/backend/parsers/windmill-parser-py-asset/Cargo.toml @@ -11,6 +11,6 @@ path = "./src/lib.rs" [dependencies] windmill-parser.workspace = true windmill-parser-sql-asset.workspace = true -rustpython-parser.workspace = true -rustpython-ast = { version = "0.4.0", features = ["visitor"] } +ruff_python_parser.workspace = true +ruff_python_ast.workspace = true anyhow.workspace = true diff --git a/backend/parsers/windmill-parser-py-asset/src/lib.rs b/backend/parsers/windmill-parser-py-asset/src/lib.rs index 9c70b93d64..2919b74607 100644 --- a/backend/parsers/windmill-parser-py-asset/src/lib.rs +++ b/backend/parsers/windmill-parser-py-asset/src/lib.rs @@ -1,5 +1,8 @@ -use rustpython_ast::{Constant, Expr, ExprConstant, Visitor}; -use rustpython_parser::{ast::Suite, Parse}; +use ruff_python_ast::{ + visitor::{walk_expr, walk_stmt, Visitor}, + Expr, ExprCall, ExprDict, FStringPartRef, InterpolatedStringElement, Keyword, Stmt, StmtAssign, +}; +use ruff_python_parser::parse_module; use std::collections::HashMap; use windmill_parser::asset_parser::{ asset_was_used, merge_assets, parse_asset_syntax, parse_pipeline_annotations, AssetKind, @@ -8,12 +11,12 @@ use windmill_parser::asset_parser::{ use AssetUsageAccessType::*; pub fn parse_assets(input: &str) -> anyhow::Result { - let ast = Suite::parse(input, "main.py") - .map_err(|e| anyhow::anyhow!("Error parsing code: {}", e.to_string()))?; + let ast = parse_module(input) + .map_err(|e| anyhow::anyhow!("Error parsing code: {}", e.to_string()))? + .into_suite(); let mut assets_finder = AssetsFinder { assets: vec![], var_identifiers: HashMap::new() }; - ast.into_iter() - .for_each(|stmt| assets_finder.visit_stmt(stmt)); + assets_finder.visit_body(&ast); for (kind, path, _) in assets_finder.var_identifiers.into_values() { // if a db = wmill.datatable() was never used (e.g db.query(...)), @@ -43,13 +46,72 @@ struct AssetsFinder { var_identifiers: HashMap, } -impl Visitor for AssetsFinder { +impl<'a> Visitor<'a> for AssetsFinder { + fn visit_stmt(&mut self, stmt: &'a Stmt) { + if let Stmt::Assign(node) = stmt { + self.track_assignment(node); + } + // Continue with generic visit to catch any other assets in the statement + walk_stmt(self, stmt); + } + + fn visit_expr(&mut self, expr: &'a Expr) { + match expr { + // A detected asset call does not recurse, + // so this is only reached when no further context was found + Expr::StringLiteral(s) => self.visit_str(s.value.to_str()), + Expr::FString(f) => { + // Literal text between two interpolations reads as one string, across the + // pieces of an implicitly concatenated f-string too. + let mut literal = String::new(); + for part in &f.value { + match part { + FStringPartRef::Literal(s) => literal.push_str(&s.value), + FStringPartRef::FString(f) => { + for element in &f.elements { + match element { + InterpolatedStringElement::Literal(l) => { + literal.push_str(&l.value) + } + InterpolatedStringElement::Interpolation(_) => { + self.visit_str(&std::mem::take(&mut literal)) + } + } + } + } + } + } + self.visit_str(&literal); + walk_expr(self, expr); + } + Expr::Call(node) => { + if self.visit_expr_call_inner(node).is_err() { + walk_expr(self, expr); + } + } + _ => walk_expr(self, expr), + } + } +} + +impl AssetsFinder { + fn visit_str(&mut self, s: &str) { + if let Some((kind, path)) = parse_asset_syntax(s, false) { + self.assets.push(ParseAssetsResult { + kind, + path: path.to_string(), + access_type: None, + columns: None, + }); + } + } + // Handle assignment statements like: x = wmill.datatable('name') - fn visit_stmt_assign(&mut self, node: rustpython_ast::StmtAssign) { + fn track_assignment(&mut self, node: &StmtAssign) { // Check if the value is a call to a tracked function if let Some(Expr::Name(expr_name)) = node.targets.first() { // Remove any tracked variables with that name in case of reassignment - let Ok(var_name) = expr_name.id.parse::(); + let var_name = expr_name.id.to_string(); let removed = self.var_identifiers.remove(&var_name); // if a db = wmill.datatable() or similar was removed, but never used (e.g db.query(...)), // we still want to register the asset as unknown access type @@ -69,59 +131,12 @@ impl Visitor for AssetsFinder { if let Some((kind, name, schema)) = self.extract_asset_from_call(&node.value) { // Track target variable - let Ok(var_name) = expr_name.id.parse::(); self.var_identifiers .insert(var_name, (kind.clone(), name.clone(), schema.clone())); } } - // Continue with generic visit to catch any other assets in the expression - self.generic_visit_stmt_assign(node); } - // visit_call_expr will not recurse if it detects an asset, - // so this will only be called when no further context was found - fn visit_expr_constant(&mut self, node: ExprConstant) { - match node.value { - Constant::Str(s) => { - if let Some((kind, path)) = parse_asset_syntax(&s, false) { - self.assets.push(ParseAssetsResult { - kind, - path: path.to_string(), - access_type: None, - columns: None, - }); - } - } - _ => self.generic_visit_expr_constant(node), - } - } - - fn visit_expr_call(&mut self, node: rustpython_ast::ExprCall) { - match self.visit_expr_call_inner(&node) { - Ok(_) => {} - Err(_) => { - // Check keyword arguments for assets before falling back to generic visit - for keyword in &node.keywords { - if let Expr::Constant(ExprConstant { value: Constant::Str(s), .. }) = - &keyword.value - { - if let Some((kind, path)) = parse_asset_syntax(s, false) { - self.assets.push(ParseAssetsResult { - kind, - path: path.to_string(), - access_type: None, - columns: None, - }); - } - } - } - self.generic_visit_expr_call(node); - } - } - } -} - -impl AssetsFinder { /// Extract asset info from calls like wmill.datatable('name'), wmill.ducklake('name'), etc. fn extract_asset_from_call( &self, @@ -147,18 +162,20 @@ impl AssetsFinder { // Get the first argument (the asset name) let name = call + .arguments .args .first() .or_else(|| { // Try keyword argument 'name' or 'path' - call.keywords + call.arguments + .keywords .iter() .find(|kw| matches!(kw.arg.as_deref(), Some("name") | Some("path"))) .map(|kw| &kw.value) }) .and_then(|first_arg| { - if let Expr::Constant(ExprConstant { value: Constant::Str(name), .. }) = first_arg { - Some(name.clone()) + if let Expr::StringLiteral(name) = first_arg { + Some(name.value.to_str().to_string()) } else { None } @@ -179,44 +196,39 @@ impl AssetsFinder { Some((kind, name, schema)) } - fn visit_expr_call_inner(&mut self, node: &rustpython_ast::ExprCall) -> Result<(), ()> { + fn visit_expr_call_inner(&mut self, node: &ExprCall) -> Result<(), ()> { let ident: String = node .func .as_name_expr() - .and_then(|o| o.id.parse().ok()) + .map(|o| o.id.to_string()) .or_else(|| { node.func .as_attribute_expr() - .and_then(|attr| attr.attr.parse().ok()) + .map(|attr| attr.attr.to_string()) }) .ok_or(())?; // `obj_name` is the object (receiver) - i.e., `obj` in `obj.method()` - let obj_name: String = if let Expr::Attribute(rustpython_ast::ExprAttribute { - value, .. - }) = node.func.as_ref() - { - if let Expr::Name(expr_name) = value.as_ref() { - expr_name.id.parse().map_err(|_| ())? - } else { - return Err(()); - } - } else { - return Err(()); - }; + let obj_name: String = node + .func + .as_attribute_expr() + .and_then(|attr| attr.value.as_name_expr()) + .map(|name| name.id.to_string()) + .ok_or(())?; if obj_name == "wmill" { // Continue } else if let Some((kind, ref path, ref schema)) = self.var_identifiers.get(&obj_name) { if ident == "query" { - let expr_name = node.args.get(0).or_else(|| { - node.keywords + let expr_name = node.arguments.args.get(0).or_else(|| { + node.arguments + .keywords .iter() .find(|kw| kw.arg.as_deref() == Some("name")) .map(|kw| &kw.value) }); let sql = match expr_name { - Some(Expr::Constant(ExprConstant { value: Constant::Str(sql), .. })) => sql, + Some(Expr::StringLiteral(sql)) => sql.value.to_str(), _ => return Err(()), }; // We use the SQL parser to detect RW, specific tables, etc. @@ -224,7 +236,7 @@ impl AssetsFinder { *kind, path, schema.as_deref(), - &sql, + sql, ); match sql_assets { Ok(Some(sql_assets)) => self.assets.extend(sql_assets), @@ -252,9 +264,10 @@ impl AssetsFinder { }; let arg_val = match arg { - Arg(i, name) => node.args.get(i).or_else(|| { + Arg(i, name) => node.arguments.args.get(i).or_else(|| { // Get arg by name - node.keywords + node.arguments + .keywords .iter() .find(|kw| kw.arg.as_deref() == Some(name)) .map(|kw| &kw.value) @@ -270,8 +283,9 @@ impl AssetsFinder { self.assets .push(ParseAssetsResult { kind, path, access_type, columns: None }); } - Some(Expr::Constant(ExprConstant { value: Constant::Str(value), .. })) => { - let path = parse_asset_syntax(&value, false) + Some(Expr::StringLiteral(value)) => { + let value = value.value.to_str(); + let path = parse_asset_syntax(value, false) .map(|(_, p)| p) .unwrap_or(std::borrow::Cow::Borrowed(value)); self.assets.push(ParseAssetsResult { @@ -291,26 +305,24 @@ impl AssetsFinder { struct Arg(usize, &'static str); /// Extract a string-literal keyword argument, e.g. `s3="value"` in a call. -fn keyword_str_value(keywords: &[rustpython_ast::Keyword], name: &str) -> Option { +fn keyword_str_value(keywords: &[Keyword], name: &str) -> Option { keywords .iter() .find(|kw| kw.arg.as_deref() == Some(name)) .and_then(|kw| match &kw.value { - Expr::Constant(ExprConstant { value: Constant::Str(s), .. }) => Some(s.clone()), + Expr::StringLiteral(s) => Some(s.value.to_str().to_string()), _ => None, }) } /// Extract a string-literal value from a dict literal, e.g. `{"s3": "value"}`. -fn dict_str_value(dict: &rustpython_ast::ExprDict, name: &str) -> Option { - dict.keys +fn dict_str_value(dict: &ExprDict, name: &str) -> Option { + dict.items .iter() - .zip(dict.values.iter()) - .find_map(|(key, value)| match (key.as_ref()?, value) { - ( - Expr::Constant(ExprConstant { value: Constant::Str(k), .. }), - Expr::Constant(ExprConstant { value: Constant::Str(v), .. }), - ) if k.as_str() == name => Some(v.clone()), + .find_map(|item| match (item.key.as_ref()?, &item.value) { + (Expr::StringLiteral(k), Expr::StringLiteral(v)) if k.value.to_str() == name => { + Some(v.value.to_str().to_string()) + } _ => None, }) } @@ -328,12 +340,13 @@ fn dict_str_value(dict: &rustpython_ast::ExprDict, name: &str) -> Option /// matches the TS object form and the `# on s3:///…` trigger form exactly. fn s3_object_arg_path(expr: &Expr) -> Option { let uri = match expr { - Expr::Constant(ExprConstant { value: Constant::Str(s), .. }) => { + Expr::StringLiteral(s) => { + let s = s.value.to_str(); match s .strip_prefix("s3://") .and_then(|rest| rest.split_once('/')) { - Some((_, key)) if !key.is_empty() => s.clone(), + Some((_, key)) if !key.is_empty() => s.to_string(), _ => return None, } } @@ -347,8 +360,9 @@ fn s3_object_arg_path(expr: &Expr) -> Option { if func_name != "S3Object" { return None; } - let key = keyword_str_value(&call.keywords, "s3")?; - let storage = keyword_str_value(&call.keywords, "storage").unwrap_or_default(); + let key = keyword_str_value(&call.arguments.keywords, "s3")?; + let storage = + keyword_str_value(&call.arguments.keywords, "storage").unwrap_or_default(); format!("s3://{storage}/{key}") } Expr::Dict(dict) => { @@ -822,4 +836,62 @@ def main(): },]) ); } + + #[test] + fn test_py_asset_parser_recent_syntax() { + let input = r#" +import wmill +lazy import polars + +type Rows = list[dict] + +def main[T](name: str): + try: + with wmill.load_s3_file_reader('s3:///in.csv') as f: + f.read() + except ValueError, TypeError: + pass + greeting = t"hello {name}" + return f"{wmill.get_resource("u/user/pg")["host"]}" +"#; + let s = parse_assets(input).map(|o| o.assets); + assert_eq!( + s.map_err(|e| e.to_string()), + Ok(vec![ + ParseAssetsResult { + kind: AssetKind::S3Object, + path: "/in.csv".to_string(), + access_type: Some(R), + columns: None, + }, + ParseAssetsResult { + kind: AssetKind::Resource, + path: "u/user/pg".to_string(), + access_type: None, + columns: None, + }, + ]) + ); + } + + #[test] + fn test_py_asset_parser_schema_qualified_db_without_table() { + let input = r#" +import wmill +def main(): + db = wmill.datatable('dt:sch') + db.query("ATTACH 'ducklake://other' AS o") + db.query("SELECT * FROM t") +"#; + let s = parse_assets(input).map(|o| o.assets); + assert_eq!( + s.map_err(|e| e.to_string()) + .map(|a| a.into_iter().map(|a| (a.kind, a.path)).collect::>()), + Ok(vec![ + (AssetKind::DataTable, "dt".to_string()), + (AssetKind::DataTable, "dt/sch.t".to_string()), + (AssetKind::Ducklake, "other".to_string()), + ]) + ); + } } diff --git a/backend/parsers/windmill-parser-py-imports/Cargo.toml b/backend/parsers/windmill-parser-py-imports/Cargo.toml index 565e1422bb..2ef83f5ac9 100644 --- a/backend/parsers/windmill-parser-py-imports/Cargo.toml +++ b/backend/parsers/windmill-parser-py-imports/Cargo.toml @@ -23,7 +23,8 @@ tracing.workspace = true [dependencies] windmill-parser.workspace = true -rustpython-parser.workspace = true +ruff_python_parser.workspace = true +ruff_python_ast.workspace = true phf.workspace = true itertools.workspace = true serde_json.workspace = true diff --git a/backend/parsers/windmill-parser-py-imports/src/lib.rs b/backend/parsers/windmill-parser-py-imports/src/lib.rs index de7f51bc79..5c8de9790f 100644 --- a/backend/parsers/windmill-parser-py-imports/src/lib.rs +++ b/backend/parsers/windmill-parser-py-imports/src/lib.rs @@ -13,9 +13,9 @@ use async_recursion::async_recursion; use itertools::Itertools; use lazy_static::lazy_static; #[cfg(not(target_arch = "wasm32"))] -use std::str::FromStr; -#[cfg(not(target_arch = "wasm32"))] use std::collections::HashMap; +#[cfg(not(target_arch = "wasm32"))] +use std::str::FromStr; use mapping::{FULL_IMPORTS_MAP, SHORT_IMPORTS_MAP}; #[cfg(not(target_arch = "wasm32"))] @@ -23,11 +23,8 @@ use regex::Regex; #[cfg(target_arch = "wasm32")] use regex_lite::Regex; -use rustpython_parser::{ - ast::{Stmt, StmtImport, StmtImportFrom, Suite}, - text_size::TextRange, - Parse, -}; +use ruff_python_ast::{Stmt, StmtImport, StmtImportFrom}; +use ruff_python_parser::parse_module; #[cfg(not(target_arch = "wasm32"))] use sqlx::{Pool, Postgres}; #[cfg(not(target_arch = "wasm32"))] @@ -160,6 +157,7 @@ pub struct ImportPin { } pub fn parse_code_for_imports(code: &str, path: &str) -> anyhow::Result> { + let full_code = code; // Use regex to safely find the main function definition let mut code = DEF_MAIN_RE .split(code) @@ -186,16 +184,22 @@ pub fn parse_code_for_imports(code: &str, path: &str) -> anyhow::Result (code, parsed.into_suite()), + // The split lands on any line reading like the definition of `main`, one inside a + // string included: what precedes it is then not a module, while the whole file is. + Err(e) => match parse_module(full_code) { + Ok(parsed) => (full_code.to_string(), parsed.into_suite()), + Err(_) => anyhow::bail!("Error parsing code for imports: {}", e.to_string()), + }, + }; // Note: We're still using the original code for finding pins, // as the TextRange values from the parsed AST would be based on code_with_fake_main // but we want to match against the original code - let find_pin = |range: TextRange, key: String| { + let find_pin = |stmt_end: usize, key: String| { let hs = code .chars() - .skip(range.end().to_usize()) + .skip(stmt_end) .take_while(|e| *e != '\n') .collect::(); @@ -226,9 +230,9 @@ pub fn parse_code_for_imports(code: &str, path: &str) -> anyhow::Result = ast .into_iter() .filter_map(|x| match x { - Stmt::Import(StmtImport { names, range }) => names + Stmt::Import(StmtImport { names, range, .. }) => names .get(0) - .and_then(|al| find_pin(range, al.name.to_string())) + .and_then(|al| find_pin(range.end().to_usize(), al.name.to_string())) .or(Some( names .into_iter() @@ -239,15 +243,11 @@ pub fn parse_code_for_imports(code: &str, path: &str) -> anyhow::Result>(), )), - Stmt::ImportFrom(StmtImportFrom { level: Some(i), module, .. }) if i.to_u32() > 0 => { - Some(process_import( - module.map(|x| x.to_string()), - path, - i.to_usize(), - )) - } + Stmt::ImportFrom(StmtImportFrom { level, module, .. }) if level > 0 => Some( + process_import(module.map(|x| x.to_string()), path, level as usize), + ), Stmt::ImportFrom(StmtImportFrom { level: _, module, range, .. }) => find_pin( - range, + range.end().to_usize(), module.clone().map(|x| x.to_string()).unwrap_or_default(), ) .or(Some(process_import(module.map(|x| x.to_string()), path, 0))), @@ -351,13 +351,16 @@ async fn parse_python_imports_inner( temp_script_refs: &Option>, ) -> error::Result> { tracing::debug!("Parsing python imports for path: {}", path); - let PythonAnnotations { py310, py311, py312, py313, .. } = PythonAnnotations::parse(&code); + let PythonAnnotations { py310, py311, py312, py313, py314, py315, .. } = + PythonAnnotations::parse(&code); tracing::debug!( - "Found python annotations - py310: {}, py311: {}, py312: {}, py313: {}", + "Found python annotations - py310: {}, py311: {}, py312: {}, py313: {}, py314: {}, py315: {}", py310, py311, py312, - py313 + py313, + py314, + py315 ); let mut push_version_specifiers = |perform, unparsed: String| -> error::Result<()> { @@ -373,6 +376,8 @@ async fn parse_python_imports_inner( push_version_specifiers(py311, "==3.11.*".to_owned())?; push_version_specifiers(py312, "==3.12.*".to_owned())?; push_version_specifiers(py313, "==3.13.*".to_owned())?; + push_version_specifiers(py314, "==3.14.*".to_owned())?; + push_version_specifiers(py315, "==3.15.*".to_owned())?; for x in code.lines() { if x.starts_with("# py:") || x.starts_with("#py:") { @@ -513,8 +518,14 @@ async fn parse_python_imports_inner( let mut nested = match n { NImport::Relative(rpath) => { // First try to get content from temp_script_refs cache if available - let code_from_cache = if let Some(hash) = temp_script_refs.as_ref().and_then(|dt| dt.get(&rpath)) { - tracing::debug!("Found relative import '{}' in temp_script_refs with hash '{}'", rpath, hash); + let code_from_cache = if let Some(hash) = + temp_script_refs.as_ref().and_then(|dt| dt.get(&rpath)) + { + tracing::debug!( + "Found relative import '{}' in temp_script_refs with hash '{}'", + rpath, + hash + ); match windmill_common::cache::raw_script_temp::load(hash.clone(), db).await { Ok(content) => Some(content), Err(e) => { @@ -529,19 +540,17 @@ async fn parse_python_imports_inner( // Use cached content if available, otherwise fall back to deployed script let code = match code_from_cache { Some(content) => content, - None => { - sqlx::query_scalar!( - r#" + None => sqlx::query_scalar!( + r#" SELECT content FROM script WHERE path = $1 AND workspace_id = $2 AND archived = false ORDER BY created_at DESC LIMIT 1 "#, - &rpath, - w_id - ) - .fetch_optional(db) - .await? - .unwrap_or_else(|| "".to_string()) - } + &rpath, + w_id + ) + .fetch_optional(db) + .await? + .unwrap_or_else(|| "".to_string()), }; if already_visited.contains(&rpath) { diff --git a/backend/parsers/windmill-parser-py-imports/src/mapping.rs b/backend/parsers/windmill-parser-py-imports/src/mapping.rs index 6246a1f156..90f3fab99c 100644 --- a/backend/parsers/windmill-parser-py-imports/src/mapping.rs +++ b/backend/parsers/windmill-parser-py-imports/src/mapping.rs @@ -383,5 +383,10 @@ pub static SHORT_IMPORTS_MAP: PyMap = phf_map! { "vt" => "vt-py", "grpc" => "grpcio", "serpapi" => "google-search-results", + "pptx" => "python-pptx", + "OpenSSL" => "pyOpenSSL", + "win32api" => "pywin32", + "ruamel" => "ruamel.yaml", + "speech-recognition" => "SpeechRecognition", // Add new entry here ^ }; diff --git a/backend/parsers/windmill-parser-py-imports/tests/tests.rs b/backend/parsers/windmill-parser-py-imports/tests/tests.rs index c853c8e5b4..9086b29d47 100644 --- a/backend/parsers/windmill-parser-py-imports/tests/tests.rs +++ b/backend/parsers/windmill-parser-py-imports/tests/tests.rs @@ -1,7 +1,9 @@ mod tests { use sqlx::{Pool, Postgres}; - use windmill_parser_py_imports::parse_python_imports; + use windmill_parser_py_imports::{ + parse_code_for_imports, parse_python_imports, ImportPin, NImport, + }; #[sqlx::test(fixtures("base"))] async fn test_parse_python_imports(db: Pool) -> anyhow::Result<()> { @@ -117,4 +119,87 @@ def main(): Ok(()) } + + fn auto(pkg: &str) -> NImport { + NImport::Auto { pkg: pkg.to_string(), key: None } + } + + #[test] + fn test_parse_lazy_imports() -> anyhow::Result<()> { + let code = " +lazy import requests +lazy import numpy # pin: numpy==2.1.0 +lazy from pandas import DataFrame +lazy from f.shared import helper +lazy = 1 + +def main(): + pass +"; + assert_eq!( + parse_code_for_imports(code, "f/foo/bar")?, + vec![ + NImport::Pin { + pins: vec![ImportPin { + pkg: "numpy==2.1.0".to_string(), + path: "f/foo/bar".to_string() + }], + key: "numpy".to_string() + }, + auto("pandas"), + auto("requests"), + NImport::Relative("f/shared".to_string()), + ] + ); + Ok(()) + } + + #[test] + fn test_parse_imports_recent_syntax_before_main() -> anyhow::Result<()> { + let code = r#" +import requests + +type Alias = list[int] +NAME = "x" +D = {"a": 1} +L = [[1], [2]] +TEMPLATE = t"hello {NAME}" +PEP701 = f"{D["a"]} {'\n'.join(["x"])}" +PEP798 = [*a for a in L] + +try: + import ujson +except ImportError, ValueError: + pass + +def main(): + pass +"#; + assert_eq!( + parse_code_for_imports(code, "f/foo/bar")?, + vec![auto("requests")] + ); + Ok(()) + } + + #[test] + fn test_parse_imports_main_lookalike_in_docstring() -> anyhow::Result<()> { + for example in ["def main[T](x: T):", "def main(x):"] { + let code = format!( + "\"\"\"Usage example:\n{example}\n return x\n\"\"\"\nimport requests # pin: requests==2.32.0\n\ndef main(value: str):\n return value\n" + ); + assert_eq!( + parse_code_for_imports(&code, "f/foo/bar")?, + vec![NImport::Pin { + pins: vec![ImportPin { + pkg: "requests==2.32.0".to_string(), + path: "f/foo/bar".to_string() + }], + key: "requests".to_string() + }], + "{code}" + ); + } + Ok(()) + } } diff --git a/backend/parsers/windmill-parser-py/Cargo.toml b/backend/parsers/windmill-parser-py/Cargo.toml index 0c50e0f8d8..6971fe8fcd 100644 --- a/backend/parsers/windmill-parser-py/Cargo.toml +++ b/backend/parsers/windmill-parser-py/Cargo.toml @@ -10,8 +10,8 @@ path = "./src/lib.rs" [dependencies] windmill-parser.workspace = true -rustpython-parser.workspace = true +ruff_python_parser.workspace = true +ruff_python_ast.workspace = true itertools.workspace = true serde_json.workspace = true anyhow.workspace = true -rustpython-ast = { version = "0.4.0", features = ["visitor"] } diff --git a/backend/parsers/windmill-parser-py/src/lib.rs b/backend/parsers/windmill-parser-py/src/lib.rs index 086a354d02..5a9eabc33e 100644 --- a/backend/parsers/windmill-parser-py/src/lib.rs +++ b/backend/parsers/windmill-parser-py/src/lib.rs @@ -13,13 +13,11 @@ use itertools::Itertools; use serde_json::json; use windmill_parser::{json_to_typ, Arg, MainArgSignature, ObjectType, Typ}; -use rustpython_parser::{ - ast::{ - Constant, Expr, ExprAttribute, ExprConstant, ExprDict, ExprList, ExprName, Stmt, - StmtAssign, StmtAsyncFunctionDef, StmtClassDef, StmtFunctionDef, Suite, - }, - Parse, +use ruff_python_ast::{ + Expr, ExprAttribute, ExprDict, ExprList, ExprName, Number, Parameters, Stmt, StmtAssign, + StmtClassDef, StmtFunctionDef, }; +use ruff_python_parser::parse_module; pub mod pydantic_parser; @@ -81,22 +79,24 @@ fn should_parse_for_models(code: &str) -> bool { || code.contains("import dataclasses") } +/// What follows `def ` or `async def ` on a line opening that function's +/// parameters or type parameters. +fn strip_def<'a>(line: &'a str, name: &str) -> Option<&'a str> { + let rest = line + .strip_prefix("async def ") + .or_else(|| line.strip_prefix("def "))? + .strip_prefix(name)?; + (rest.starts_with('(') || rest.starts_with('[')).then_some(rest) +} + fn filter_non_main(code: &str, main_name: &str) -> String { - let def_main = format!("def {}(", main_name); - let async_def_main = format!("async def {}(", main_name); let mut filtered_code = String::new(); let mut code_iter = code.split("\n"); let mut remaining: String = String::new(); while let Some(line) = code_iter.next() { - if line.starts_with(&async_def_main) { - filtered_code += &async_def_main; - remaining += line.strip_prefix(&async_def_main).unwrap(); - remaining += "\n"; - remaining += &code_iter.join("\n"); - break; - } else if line.starts_with(&def_main) { - filtered_code += &def_main; - remaining += line.strip_prefix(&def_main).unwrap(); + if let Some(rest) = strip_def(line, main_name) { + filtered_code += &line[..line.len() - rest.len()]; + remaining += rest; remaining += "\n"; remaining += &code_iter.join("\n"); break; @@ -106,6 +106,26 @@ fn filter_non_main(code: &str, main_name: &str) -> String { return String::new(); } let mut chars = remaining.chars(); + + // PEP 695 type parameters, `def main[T](x: T)`, sit between the name and the parameters. + if remaining.starts_with('[') { + let mut open_brackets = 0; + for c in chars.by_ref() { + if c == '[' { + open_brackets += 1; + } else if c == ']' { + open_brackets -= 1; + } + filtered_code.push(c); + if open_brackets == 0 { + break; + } + } + } + if chars.next() != Some('(') { + return String::new(); + } + filtered_code.push('('); let mut open_parens = 1; while let Some(c) = chars.next() { @@ -183,15 +203,15 @@ fn extract_code_metadata(code: &str, main_name: &str) -> CodeMetadata { let filtered_code = filter_relevant_statements(code); - let ast = match Suite::parse(&filtered_code, "main.py") { - Ok(ast) => ast, + let ast = match parse_module(&filtered_code) { + Ok(parsed) => parsed.into_suite(), Err(_) => return CodeMetadata { enums, descriptions }, }; for stmt in ast { match stmt { - Stmt::ClassDef(StmtClassDef { name, body, bases, .. }) if has_enum => { - let is_enum = bases.iter().any(|base| { + Stmt::ClassDef(class_def) if has_enum => { + let is_enum = class_def.bases().iter().any(|base| { matches!(base, Expr::Name(ExprName { id, .. }) if id == "Enum" || id == "IntEnum" || id == "StrEnum" || id == "Flag" || id == "IntFlag") @@ -201,17 +221,15 @@ fn extract_code_metadata(code: &str, main_name: &str) -> CodeMetadata { let mut values = Vec::new(); let mut members = HashMap::new(); + let StmtClassDef { name, body, .. } = class_def; for item in body { if let Stmt::Assign(StmtAssign { targets, value, .. }) = item { if let Some(Expr::Name(ExprName { id: target_name, .. })) = targets.first() { if !target_name.starts_with('_') { - if let Expr::Constant(ExprConstant { - value: Constant::Str(val), - .. - }) = value.as_ref() - { + if let Expr::StringLiteral(val) = value.as_ref() { + let val = val.value.to_str(); values.push(val.to_string()); members.insert(target_name.to_string(), val.to_string()); } @@ -225,15 +243,13 @@ fn extract_code_metadata(code: &str, main_name: &str) -> CodeMetadata { } } } - Stmt::FunctionDef(StmtFunctionDef { name: func_name, body, .. }) if has_docstring => { - if &func_name == main_name { + Stmt::FunctionDef(StmtFunctionDef { + name: func_name, body, is_async: false, .. + }) if has_docstring => { + if func_name.as_str() == main_name { if let Some(Stmt::Expr(expr_stmt)) = body.first() { - if let Expr::Constant(ExprConstant { - value: Constant::Str(docstring), - .. - }) = expr_stmt.value.as_ref() - { - descriptions = parse_docstring_args(docstring); + if let Expr::StringLiteral(docstring) = expr_stmt.value.as_ref() { + descriptions = parse_docstring_args(docstring.value.to_str()); } } } @@ -294,6 +310,27 @@ fn parse_docstring_args(docstring: &str) -> HashMap { descriptions } +/// A line that reads as the definition of `main_name` can also sit inside a string, a docstring +/// showing an example for instance. When several lines do, only the parsed module tells which +/// one is the function. +fn main_params_among_lookalikes(code: &str, main_name: &str) -> Option { + code.split('\n') + .filter(|line| strip_def(line, main_name).is_some()) + .nth(1)?; + parse_module(code) + .ok()? + .into_suite() + .into_iter() + .find_map(|x| match x { + Stmt::FunctionDef(StmtFunctionDef { name, parameters, .. }) + if name.as_str() == main_name => + { + Some(*parameters) + } + _ => None, + }) +} + /// skip_params is a micro optimization for when we just want to find the main /// function without parsing all the params. pub fn parse_python_signature( @@ -310,41 +347,39 @@ pub fn parse_python_signature( // - If no models: parse only the filtered main function let (params, module) = if should_parse_for_models(code) { // Parse full code once for both Pydantic detection and signature extraction - let ast = Suite::parse(code, "main.py") - .map_err(|e| anyhow::anyhow!("Error parsing code: {}", e.to_string()))?; + let ast = parse_module(code) + .map_err(|e| anyhow::anyhow!("Error parsing code: {}", e.to_string()))? + .into_suite(); // Extract main function from full AST let params = ast.iter().find_map(|x| match x { - Stmt::FunctionDef(StmtFunctionDef { name, args, .. }) if name == &main_name => { - Some(args.as_ref().clone()) - } - Stmt::AsyncFunctionDef(StmtAsyncFunctionDef { name, args, .. }) - if name == &main_name => + Stmt::FunctionDef(StmtFunctionDef { name, parameters, .. }) + if name.as_str() == main_name => { - Some(args.as_ref().clone()) + Some(parameters.as_ref().clone()) } _ => None, }); // Keep AST for Pydantic/dataclass detection (params, Some(ast)) + } else if let Some(params) = main_params_among_lookalikes(code, &main_name) { + (Some(params), None) } else { // No models detected - parse only the filtered main function let filtered_code = filter_non_main(code, &main_name); if filtered_code.is_empty() { (None, None) } else { - let ast = Suite::parse(&filtered_code, "main.py") - .map_err(|e| anyhow::anyhow!("Error parsing code: {}", e.to_string()))?; + let ast = parse_module(&filtered_code) + .map_err(|e| anyhow::anyhow!("Error parsing code: {}", e.to_string()))? + .into_suite(); let params = ast.into_iter().find_map(|x| match x { - Stmt::FunctionDef(StmtFunctionDef { name, args, .. }) if &name == &main_name => { - Some(*args) - } - Stmt::AsyncFunctionDef(StmtAsyncFunctionDef { name, args, .. }) - if &name == &main_name => + Stmt::FunctionDef(StmtFunctionDef { name, parameters, .. }) + if name.as_str() == main_name => { - Some(*args) + Some(*parameters) } _ => None, }); @@ -378,8 +413,6 @@ pub fn parse_python_signature( if !skip_params && params.is_some() { let params = params.unwrap(); - let def_arg_start = params.args.len() - params.defaults().count(); - // Two-pass approach for lazy metadata extraction: // Pass 1: Parse types without enum info to determine if metadata is needed // Pass 2: Re-parse unknown types with metadata only if necessary @@ -392,11 +425,11 @@ pub fn parse_python_signature( .iter() .enumerate() .map(|(i, x)| { - let arg_name = x.as_arg().arg.to_string(); + let arg_name = x.parameter.name.to_string(); let (typ, has_default) = x - .as_arg() + .parameter .annotation - .as_ref() + .as_deref() .map_or((Typ::Unknown, false), |e| { parse_expr(e, &empty_enums, module_ref) }); @@ -423,21 +456,16 @@ pub fn parse_python_signature( .into_iter() .map(|(i, arg_name, mut typ, mut has_default)| { if matches!(typ, Typ::Resource(_)) && !metadata.enums.is_empty() { - if let Some(annotation) = params.args[i].as_arg().annotation.as_ref() { + if let Some(annotation) = params.args[i].parameter.annotation.as_deref() { (typ, has_default) = parse_expr(annotation, &metadata.enums, module_ref); } } - let default = if i >= def_arg_start { - params - .defaults() - .nth(i - def_arg_start) - .map(|expr| to_value(expr, &metadata.enums)) - .flatten() - } else { - None - }; + let default = params.args[i] + .default + .as_deref() + .and_then(|expr| to_value(expr, &metadata.enums)); let should_get_type_from_default = match &typ { Typ::Unknown => true, @@ -465,10 +493,10 @@ pub fn parse_python_signature( // debounce accumulation support. Falls back to docstring // description if no union array pattern is found. let union_otyp = params.args[i] - .as_arg() + .parameter .annotation - .as_ref() - .and_then(|ann| detect_py_union_array_otyp(ann.as_ref())); + .as_deref() + .and_then(detect_py_union_array_otyp); Arg { otyp: union_otyp.or_else(|| { @@ -509,12 +537,8 @@ pub fn parse_python_signature( } } -fn parse_expr( - e: &Box, - enums: &HashMap, - module: Option<&[Stmt]>, -) -> (Typ, bool) { - match e.as_ref() { +fn parse_expr(e: &Expr, enums: &HashMap, module: Option<&[Stmt]>) -> (Typ, bool) { + match e { Expr::Name(ExprName { id, .. }) => (parse_typ(id.as_ref(), enums, module), false), Expr::Attribute(x) => { if let Some(name) = x.value.as_name_expr() { @@ -531,12 +555,9 @@ fn parse_expr( } } Expr::BinOp(x) => { - if matches!( - x.right.as_ref(), - Expr::Constant(ExprConstant { value: Constant::None, .. }) - ) { + if x.right.is_none_literal_expr() { (parse_expr(&x.left, enums, module).0, true) - } else if detect_py_union_array_otyp(e.as_ref()).is_some() { + } else if detect_py_union_array_otyp(e).is_some() { // T | list[T] — parsed type is Unknown; otyp is set separately (Typ::Unknown, false) } else { @@ -552,7 +573,7 @@ fn parse_expr( .elts .iter() .map(|x| match x { - Expr::Constant(c) => c.value.as_str().map(|x| x.to_string()), + Expr::StringLiteral(s) => Some(s.value.to_str().to_string()), _ => None, }) .filter_map(|x| x) @@ -625,15 +646,26 @@ fn map_resource_name(x: &str) -> String { } } -fn to_value(et: &Expr, enums: &HashMap) -> Option { +fn to_value(et: &Expr, enums: &HashMap) -> Option { match et { - Expr::Constant(ExprConstant { value, .. }) => Some(constant_to_value(value)), - Expr::Dict(ExprDict { keys, values, .. }) => { - let v = keys - .into_iter() - .zip(values) - .map(|(k, v)| { - let key = k + Expr::NoneLiteral(_) => Some(json!(null)), + Expr::BooleanLiteral(b) => Some(json!(b.value)), + Expr::StringLiteral(s) => Some(json!(s.value.to_str())), + Expr::BytesLiteral(b) => Some(json!(b.value.bytes().collect::>())), + Expr::NumberLiteral(n) => Some(match &n.value { + Number::Int(i) => { + serde_json::from_str(&i.to_string()).unwrap_or(json!("invalid number")) + } + Number::Float(f) => json!(f), + Number::Complex { real, imag } => json!([real, imag]), + }), + Expr::EllipsisLiteral(_) => Some(json!("...")), + Expr::Dict(ExprDict { items, .. }) => { + let v = items + .iter() + .map(|item| { + let key = item + .key .as_ref() .map(|e| to_value(e, enums)) .flatten() @@ -642,7 +674,7 @@ fn to_value(et: &Expr, enums: &HashMap) -> Option None, }) .unwrap_or_else(|| "no_key".to_string()); - (key, to_value(&v, enums)) + (key, to_value(&item.value, enums)) }) .collect::>(); Some(json!(v)) @@ -672,20 +704,6 @@ fn to_value(et: &Expr, enums: &HashMap) -> Option serde_json::Value { - match c { - Constant::None => json!(null), - Constant::Bool(b) => json!(b), - Constant::Str(s) => json!(s), - Constant::Bytes(b) => json!(b), - Constant::Int(i) => serde_json::from_str(&i.to_string()).unwrap_or(json!("invalid number")), - Constant::Tuple(t) => json!(t.iter().map(constant_to_value).collect::>()), - Constant::Float(f) => json!(f), - Constant::Complex { real, imag } => json!([real, imag]), - Constant::Ellipsis => json!("..."), - } -} - #[cfg(test)] mod tests { @@ -1323,4 +1341,89 @@ def main(color: Color = Color.RED): ); Ok(()) } + + #[test] + fn test_parse_python_sig_type_params() -> anyhow::Result<()> { + for code in [ + "def main[T](x: str, y: int = 3):\n return x\n", + "async def main[T: (int, str), *Ts](x: str, y: int = 3):\n return x\n", + // parsed whole rather than pre-filtered down to `main` + "from dataclasses import dataclass\n\ndef main[T](x: str, y: int = 3):\n return x\n", + ] { + let result = parse_python_signature(code, None, false)?; + assert_eq!(result.auto_kind, None, "{code}"); + assert_eq!( + result + .args + .iter() + .map(|a| (a.name.as_str(), &a.typ, &a.default)) + .collect::>(), + vec![ + ("x", &Typ::Str(None), &None), + ("y", &Typ::Int, &Some(json!(3))) + ], + "{code}" + ); + } + Ok(()) + } + + // A script with a model class is parsed whole, so syntax anywhere in it reaches the parser. + #[test] + fn test_parse_python_sig_recent_syntax() -> anyhow::Result<()> { + let code = r#" +from dataclasses import dataclass +lazy import requests +lazy from pandas import DataFrame + +type Alias = list[int] + +@dataclass +class Point: + x: int + +try: + import a +except ImportError, ValueError: + pass + +D = {"a": 1} +L = [[1], [2]] +PEP701 = f"{D["a"]} {'\n'.join(["x"])}" +PEP798 = [*a for a in L] + +def main(p: Point, name: str = "world"): + return t"hello {name}" +"#; + let result = parse_python_signature(code, None, false)?; + assert_eq!(result.args.len(), 2); + assert_eq!( + result.args[0].typ, + Typ::Object(ObjectType::new( + Some("Point".to_string()), + Some(vec![windmill_parser::ObjectProperty { + key: "x".to_string(), + typ: Box::new(Typ::Int) + }]) + )) + ); + assert_eq!(result.args[1].default, Some(json!("world"))); + Ok(()) + } + + #[test] + fn test_parse_python_sig_main_lookalike_in_docstring() -> anyhow::Result<()> { + for example in ["def main[T](x: T):", "def main(x: int, y: int):"] { + let code = format!( + "\"\"\"Usage example:\n{example}\n return x\n\"\"\"\n\ndef main(value: str):\n return value\n" + ); + let result = parse_python_signature(&code, None, false)?; + assert_eq!( + result.args.iter().map(|a| &a.name).collect::>(), + vec!["value"], + "{code}" + ); + } + Ok(()) + } } diff --git a/backend/parsers/windmill-parser-py/src/pydantic_parser.rs b/backend/parsers/windmill-parser-py/src/pydantic_parser.rs index e229d97a8a..78a0a3c354 100644 --- a/backend/parsers/windmill-parser-py/src/pydantic_parser.rs +++ b/backend/parsers/windmill-parser-py/src/pydantic_parser.rs @@ -11,8 +11,8 @@ //! This module provides functionality to detect and parse Pydantic models and Python //! dataclasses from Python AST, enabling automatic UI generation for complex data structures. -use rustpython_parser::ast::{ - Constant, Expr, ExprAttribute, ExprCall, ExprConstant, ExprName, ExprTuple, Stmt, StmtAnnAssign, +use ruff_python_ast::{ + Decorator, Expr, ExprAttribute, ExprCall, ExprName, ExprTuple, Stmt, StmtAnnAssign, }; use std::collections::HashSet; use windmill_parser::{ObjectProperty, ObjectType, Typ}; @@ -64,7 +64,7 @@ fn detect_model_type_impl( // Mark as being visited visited.insert(class_name.to_string()); - let result = if is_pydantic_base(&class_def.bases) { + let result = if is_pydantic_base(class_def.bases()) { // Pydantic BaseModel parse_model_fields(&class_def.body, class_def.name.as_str(), module, visited) } else if has_dataclass_decorator(&class_def.decorator_list) @@ -115,9 +115,9 @@ fn is_pydantic_base(bases: &[Expr]) -> bool { } /// Checks if a class has @dataclass decorator (standard library) -fn has_dataclass_decorator(decorators: &[Expr]) -> bool { +fn has_dataclass_decorator(decorators: &[Decorator]) -> bool { for decorator in decorators { - match decorator { + match &decorator.expression { // Match: @dataclass Expr::Name(ExprName { id, .. }) if id.as_str() == "dataclass" => { return true; @@ -160,9 +160,9 @@ fn has_dataclass_decorator(decorators: &[Expr]) -> bool { } /// Checks if a class has @pydantic.dataclasses.dataclass decorator (Pydantic v2) -fn has_pydantic_dataclass_decorator(decorators: &[Expr]) -> bool { +fn has_pydantic_dataclass_decorator(decorators: &[Decorator]) -> bool { for decorator in decorators { - match decorator { + match &decorator.expression { // Match: @pydantic.dataclasses.dataclass Expr::Attribute(ExprAttribute { attr, value, .. }) if attr.as_str() == "dataclass" => { if let Expr::Attribute(ExprAttribute { @@ -350,10 +350,10 @@ fn extract_field_type( // String annotations: "ForwardRef" (forward references) // Not fully supported - return Unknown with warning - Expr::Constant(ExprConstant { value: Constant::Str(s), .. }) => { + Expr::StringLiteral(s) => { eprintln!( "Forward references like \"{forward_ref}\" are not yet supported, treating as Unknown", - forward_ref = s, + forward_ref = s.value.to_str(), ); Typ::Unknown } diff --git a/backend/parsers/windmill-parser-sql-asset/src/asset_parser_utils.rs b/backend/parsers/windmill-parser-sql-asset/src/asset_parser_utils.rs index b29c6c99ac..4d77da15bb 100644 --- a/backend/parsers/windmill-parser-sql-asset/src/asset_parser_utils.rs +++ b/backend/parsers/windmill-parser-sql-asset/src/asset_parser_utils.rs @@ -20,13 +20,16 @@ pub fn parse_wmill_sdk_sql_assets( Ok(mut sql_assets) => { if let Some(schema) = schema { for asset in &mut sql_assets.assets { - if asset.kind == kind && asset.path.starts_with(asset_name) { - asset.path = format!( - "{}/{}.{}", - asset_name, - schema, - &asset.path[asset_name.len() + 1..] - ); + if asset.kind != kind { + continue; + } + // A statement can name the database alone, with no table to qualify. + let table = asset + .path + .strip_prefix(asset_name) + .and_then(|rest| rest.strip_prefix('/')); + if let Some(table) = table { + asset.path = format!("{asset_name}/{schema}.{table}"); } } } diff --git a/backend/parsers/windmill-parser-sql/src/lib.rs b/backend/parsers/windmill-parser-sql/src/lib.rs index 38f4b7a33e..341906a3e3 100644 --- a/backend/parsers/windmill-parser-sql/src/lib.rs +++ b/backend/parsers/windmill-parser-sql/src/lib.rs @@ -235,7 +235,7 @@ lazy_static::lazy_static! { // -- $name (type) = default static ref RE_ARG_DUCKDB: Regex = Regex::new(r#"(?m)^-- \$(\w+) \(([A-Za-z0-9_\[\]]+)\)(?: ?\= ?(.+))? *(?:\r|\n|$)"#).unwrap(); - static ref RE_ARG_SNOWFLAKE: Regex = Regex::new(r#"(?m)^-- \? (\w+) \((\w+)\)(?: ?\= ?(.+))? *(?:\r|\n|$)"#).unwrap(); + static ref RE_ARG_SNOWFLAKE: Regex = Regex::new(r#"(?m)^-- \? (\w+) \((\w+(?:\[\])?)\)(?: ?\= ?(.+))? *(?:\r|\n|$)"#).unwrap(); static ref RE_ARG_MSSQL: Regex = Regex::new(r#"(?m)^-- @(?:P|p)\d+ (\w+) \((\w+)\)(?: ?\= ?(.+))? *(?:\r|\n|$)"#).unwrap(); @@ -254,6 +254,12 @@ fn parsed_default(parsed_typ: &Typ, default: String) -> Option= 2 && default.starts_with("'") && default.ends_with("'") => { Some(json!(&default[1..default.len() - 1])) } + Typ::List(_) => Some( + serde_json::from_str::(&default) + .ok() + .filter(|v| v.is_array()) + .unwrap_or_else(|| json!(default)), + ), _ => Some(json!(default)), } } @@ -1039,12 +1045,18 @@ pub fn parse_duckdb_typ(typ: &str) -> Typ { } pub fn parse_snowflake_typ(typ: &str) -> Typ { + if let Some(base_typ) = typ.strip_suffix("[]") { + return Typ::List(Box::new(parse_snowflake_typ(base_typ))); + } match typ { - "varchar" => Typ::Str(None), - "binary" => Typ::Bytes, - "date" | "time" | "timestamp" => Typ::Datetime, - "int" => Typ::Int, - "float" => Typ::Float, + "array" => Typ::List(Box::new(Typ::Unknown)), + "varchar" | "text" | "string" | "char" | "character" => Typ::Str(None), + "binary" | "varbinary" => Typ::Bytes, + "date" | "time" | "datetime" | "timestamp" | "timestamp_ntz" | "timestamp_ltz" + | "timestamp_tz" => Typ::Datetime, + "int" | "integer" | "bigint" | "smallint" | "tinyint" | "byteint" => Typ::Int, + "float" | "float4" | "float8" | "double" | "double precision" | "real" | "number" + | "decimal" | "numeric" => Typ::Float, "boolean" => Typ::Bool, "s3object" => Typ::Resource("S3Object".to_string()), _ => Typ::Str(None), @@ -1589,6 +1601,39 @@ SELECT ?; Ok(()) } + #[test] + fn test_parse_snowflake_array_types() -> anyhow::Result<()> { + let code = r#" +-- ? groups (varchar[]) +-- ? ids (int[]) = [1, 2] +-- ? anything (ARRAY) +SELECT 1; +"#; + let args = parse_snowflake_sig(code)?.args; + assert_eq!(args[1].default, Some(json!([1, 2]))); + let got = args + .iter() + .map(|a| (a.name.as_str(), a.otyp.as_deref(), &a.typ)) + .collect::>(); + assert_eq!( + got, + vec![ + ( + "groups", + Some("varchar[]"), + &Typ::List(Box::new(Typ::Str(None))) + ), + ("ids", Some("int[]"), &Typ::List(Box::new(Typ::Int))), + ( + "anything", + Some("array"), + &Typ::List(Box::new(Typ::Unknown)) + ), + ] + ); + Ok(()) + } + #[test] fn test_parse_mssql_sig() -> anyhow::Result<()> { let code = r#" diff --git a/backend/parsers/windmill-parser-wac/Cargo.toml b/backend/parsers/windmill-parser-wac/Cargo.toml index d354b1c653..21ee286db9 100644 --- a/backend/parsers/windmill-parser-wac/Cargo.toml +++ b/backend/parsers/windmill-parser-wac/Cargo.toml @@ -9,8 +9,9 @@ name = "windmill_parser_wac" path = "./src/lib.rs" [dependencies] -rustpython-parser.workspace = true -rustpython-ast = { version = "0.4.0", features = ["visitor"] } +ruff_python_parser.workspace = true +ruff_python_ast.workspace = true +ruff_text_size.workspace = true swc_common.workspace = true swc_ecma_parser.workspace = true swc_ecma_ast.workspace = true diff --git a/backend/parsers/windmill-parser-wac/src/python.rs b/backend/parsers/windmill-parser-wac/src/python.rs index f91067b711..e8fb72ff2f 100644 --- a/backend/parsers/windmill-parser-wac/src/python.rs +++ b/backend/parsers/windmill-parser-wac/src/python.rs @@ -1,12 +1,12 @@ use std::collections::HashMap; -use rustpython_parser::{ - ast::{ - Expr, ExprAwait, ExprCall, ExprName, Stmt, StmtExpr, StmtFor, StmtIf, StmtReturn, StmtTry, - StmtTryStar, StmtWhile, - }, - Parse, +use ruff_python_ast::{ + CmpOp, ElifElseClause, ExceptHandler, Expr, ExprAttribute, ExprAwait, ExprCall, ExprName, + Number, Parameters, Stmt, StmtExpr, StmtFor, StmtFunctionDef, StmtIf, StmtReturn, StmtTry, + StmtWhile, }; +use ruff_python_parser::parse_module; +use ruff_text_size::{Ranged, TextSize}; use crate::dag::{DagEdge, DagNode, DagNodeType, Param, WorkflowDag}; use crate::validation::{self, CompileError}; @@ -42,9 +42,9 @@ type TaskFunctions = HashMap>; fn collect_task_functions(stmts: &[Stmt]) -> TaskFunctions { let mut tasks = HashMap::new(); for stmt in stmts { - if let Stmt::AsyncFunctionDef(func) = stmt { + if let Stmt::FunctionDef(func @ StmtFunctionDef { is_async: true, .. }) = stmt { for dec in &func.decorator_list { - match dec { + match &dec.expression { // @task (bare decorator) Expr::Name(ExprName { id, .. }) if id.as_str() == "task" => { tasks.insert(func.name.to_string(), None); @@ -68,13 +68,9 @@ fn collect_task_functions(stmts: &[Stmt]) -> TaskFunctions { if let Expr::Name(ExprName { id, .. }) = call.func.as_ref() { if id.as_str() == "task_script" || id.as_str() == "task_flow" { // Extract the path from the first positional argument - let path = call.args.first().and_then(|arg| { - if let Expr::Constant(c) = arg { - if let rustpython_parser::ast::Constant::Str(s) = &c.value { - return Some(s.to_string()); - } - } - None + let path = call.arguments.args.first().and_then(|arg| match arg { + Expr::StringLiteral(s) => Some(s.value.to_str().to_string()), + _ => None, }); // Extract variable name from target if let Some(Expr::Name(ExprName { id: var_name, .. })) = @@ -92,13 +88,11 @@ fn collect_task_functions(stmts: &[Stmt]) -> TaskFunctions { /// Extract the `path=` keyword argument from a `@task(path="...")` call. fn extract_task_path_kwarg(call: &ExprCall) -> Option { - for kw in &call.keywords { + for kw in &call.arguments.keywords { if let Some(ref arg) = kw.arg { if arg.as_str() == "path" { - if let Expr::Constant(c) = &kw.value { - if let rustpython_parser::ast::Constant::Str(s) = &c.value { - return Some(s.to_string()); - } + if let Expr::StringLiteral(s) = &kw.value { + return Some(s.value.to_str().to_string()); } } } @@ -150,10 +144,9 @@ impl WacWalker { fn line_of_expr(&self, expr: &Expr) -> usize { let offset = match expr { - Expr::Call(c) => c.range.start().to_usize(), - Expr::Await(a) => a.range.start().to_usize(), - Expr::Attribute(a) => a.range.start().to_usize(), - Expr::Name(n) => n.range.start().to_usize(), + Expr::Call(_) | Expr::Await(_) | Expr::Attribute(_) | Expr::Name(_) => { + expr.start().to_usize() + } _ => 0, }; self.line_index.line_of(offset) @@ -161,17 +154,15 @@ impl WacWalker { fn line_of_stmt(&self, stmt: &Stmt) -> usize { let offset = match stmt { - Stmt::If(s) => s.range.start().to_usize(), - Stmt::For(s) => s.range.start().to_usize(), - Stmt::While(s) => s.range.start().to_usize(), - Stmt::Return(s) => s.range.start().to_usize(), - Stmt::Expr(s) => s.range.start().to_usize(), - Stmt::Try(s) => s.range.start().to_usize(), - Stmt::TryStar(s) => s.range.start().to_usize(), - Stmt::Assign(s) => s.range.start().to_usize(), - Stmt::AnnAssign(s) => s.range.start().to_usize(), - Stmt::FunctionDef(s) => s.range.start().to_usize(), - Stmt::AsyncFunctionDef(s) => s.range.start().to_usize(), + Stmt::If(_) + | Stmt::For(_) + | Stmt::While(_) + | Stmt::Return(_) + | Stmt::Expr(_) + | Stmt::Try(_) + | Stmt::Assign(_) + | Stmt::AnnAssign(_) + | Stmt::FunctionDef(_) => stmt.start().to_usize(), _ => 0, }; self.line_index.line_of(offset) @@ -190,9 +181,7 @@ impl WacWalker { /// Check if an expression is `asyncio.gather(...)` call fn is_asyncio_gather_call(expr: &Expr) -> bool { if let Expr::Call(call) = expr { - if let Expr::Attribute(rustpython_parser::ast::ExprAttribute { value, attr, .. }) = - call.func.as_ref() - { + if let Expr::Attribute(ExprAttribute { value, attr, .. }) = call.func.as_ref() { if attr.as_str() == "gather" { if let Expr::Name(ExprName { id, .. }) = value.as_ref() { return id.as_str() == "asyncio"; @@ -222,20 +211,20 @@ impl WacWalker { fn expr_to_source(expr: &Expr) -> String { match expr { Expr::Compare(c) => { - let left = Self::expr_to_source(&c.left); - if let Some(comparator) = c.comparators.first() { + let left = Self::expr_to_source(&c.operands[0]); + if let Some(comparator) = c.comparators().first() { let right = Self::expr_to_source(comparator); let op = match c.ops.first() { - Some(rustpython_parser::ast::CmpOp::Gt) => ">", - Some(rustpython_parser::ast::CmpOp::Lt) => "<", - Some(rustpython_parser::ast::CmpOp::GtE) => ">=", - Some(rustpython_parser::ast::CmpOp::LtE) => "<=", - Some(rustpython_parser::ast::CmpOp::Eq) => "==", - Some(rustpython_parser::ast::CmpOp::NotEq) => "!=", - Some(rustpython_parser::ast::CmpOp::In) => "in", - Some(rustpython_parser::ast::CmpOp::NotIn) => "not in", - Some(rustpython_parser::ast::CmpOp::Is) => "is", - Some(rustpython_parser::ast::CmpOp::IsNot) => "is not", + Some(CmpOp::Gt) => ">", + Some(CmpOp::Lt) => "<", + Some(CmpOp::GtE) => ">=", + Some(CmpOp::LtE) => "<=", + Some(CmpOp::Eq) => "==", + Some(CmpOp::NotEq) => "!=", + Some(CmpOp::In) => "in", + Some(CmpOp::NotIn) => "not in", + Some(CmpOp::Is) => "is", + Some(CmpOp::IsNot) => "is not", None => "?", }; format!("{left} {op} {right}") @@ -253,14 +242,14 @@ impl WacWalker { format!("{value}.{}", a.attr) } Expr::Name(n) => n.id.to_string(), - Expr::Constant(c) => match &c.value { - rustpython_parser::ast::Constant::Str(s) => format!("\"{s}\""), - rustpython_parser::ast::Constant::Int(i) => i.to_string(), - rustpython_parser::ast::Constant::Float(f) => f.to_string(), - rustpython_parser::ast::Constant::Bool(b) => b.to_string(), - rustpython_parser::ast::Constant::None => "None".to_string(), - _ => "...".to_string(), + Expr::StringLiteral(s) => format!("\"{}\"", s.value.to_str()), + Expr::NumberLiteral(n) => match &n.value { + Number::Int(i) => i.to_string(), + Number::Float(f) => f.to_string(), + Number::Complex { .. } => "...".to_string(), }, + Expr::BooleanLiteral(b) => b.value.to_string(), + Expr::NoneLiteral(_) => "None".to_string(), _ => "...".to_string(), } } @@ -279,8 +268,16 @@ impl WacWalker { match stmt { Stmt::Expr(StmtExpr { value, .. }) => self.expr_contains_step(value), Stmt::Assign(a) => self.expr_contains_step(&a.value), - Stmt::If(s) => self.body_contains_step(&s.body) || self.body_contains_step(&s.orelse), - Stmt::For(s) => self.body_contains_step(&s.body) || self.body_contains_step(&s.orelse), + Stmt::If(s) => { + self.body_contains_step(&s.body) + || s.elif_else_clauses + .iter() + .any(|clause| self.body_contains_step(&clause.body)) + } + // `async for` is not modelled: its body is skipped like any unsupported statement + Stmt::For(s) if !s.is_async => { + self.body_contains_step(&s.body) || self.body_contains_step(&s.orelse) + } Stmt::While(s) => { self.body_contains_step(&s.body) || self.body_contains_step(&s.orelse) } @@ -289,19 +286,7 @@ impl WacWalker { || self.body_contains_step(&s.orelse) || self.body_contains_step(&s.finalbody) || s.handlers.iter().any(|h| match h { - rustpython_parser::ast::ExceptHandler::ExceptHandler(eh) => { - self.body_contains_step(&eh.body) - } - }) - } - Stmt::TryStar(s) => { - self.body_contains_step(&s.body) - || self.body_contains_step(&s.orelse) - || self.body_contains_step(&s.finalbody) - || s.handlers.iter().any(|h| match h { - rustpython_parser::ast::ExceptHandler::ExceptHandler(eh) => { - self.body_contains_step(&eh.body) - } + ExceptHandler::ExceptHandler(eh) => self.body_contains_step(&eh.body), }) } Stmt::Return(_) => false, @@ -319,11 +304,12 @@ impl WacWalker { match expr { Expr::Await(ExprAwait { value, .. }) => self.expr_contains_step(value), Expr::Call(call) => { - if self.is_task_fn_call(&Expr::Call(call.clone())) { - return true; - } - if Self::is_asyncio_gather_call(&Expr::Call(call.clone())) { - return call.args.iter().any(|a| self.expr_contains_step(a)); + if Self::is_asyncio_gather_call(expr) { + return call + .arguments + .args + .iter() + .any(|a| self.expr_contains_step(a)); } false } @@ -370,12 +356,11 @@ impl WacWalker { Stmt::Expr(StmtExpr { value, .. }) => self.walk_expr_stmt(value), Stmt::Assign(a) => self.walk_expr_stmt(&a.value), Stmt::If(if_stmt) => self.walk_if(if_stmt), - Stmt::For(for_stmt) => self.walk_for(for_stmt), + Stmt::For(for_stmt) if !for_stmt.is_async => self.walk_for(for_stmt), Stmt::While(while_stmt) => self.walk_while(while_stmt), Stmt::Try(try_stmt) => self.walk_try(try_stmt), - Stmt::TryStar(try_stmt) => self.walk_try_star(try_stmt), Stmt::Return(ret) => self.walk_return(ret), - Stmt::FunctionDef(_) | Stmt::AsyncFunctionDef(_) => { + Stmt::FunctionDef(_) => { if self.stmt_contains_step(stmt) { self.errors.push(validation::error_step_in_nested_function( self.line_of_stmt(stmt), @@ -392,7 +377,7 @@ impl WacWalker { if let Expr::Await(ExprAwait { value, .. }) = expr { // await task_fn(...) if let Expr::Call(call) = value.as_ref() { - if self.is_task_fn_call(&Expr::Call(call.clone())) { + if self.is_task_fn_call(value) { return self.emit_step(call, expr); } // Check for SDK-level calls: step(), sleep(), wait_for_approval() @@ -430,15 +415,12 @@ impl WacWalker { "step" => { // step("name", fn) — extract the name from the first string argument let name = call + .arguments .args .first() - .and_then(|arg| { - if let Expr::Constant(c) = arg { - if let rustpython_parser::ast::Constant::Str(s) = &c.value { - return Some(s.to_string()); - } - } - None + .and_then(|arg| match arg { + Expr::StringLiteral(s) => Some(s.value.to_str().to_string()), + _ => None, }) .unwrap_or_else(|| "step".to_string()); let id = self.next_id(); @@ -452,6 +434,7 @@ impl WacWalker { } "sleep" => { let seconds = call + .arguments .args .first() .map(|arg| Self::expr_to_source(arg)) @@ -517,10 +500,10 @@ impl WacWalker { }); let mut step_ids = Vec::new(); - for arg in &gather_call.args { + for arg in &gather_call.arguments.args { // Each arg should be task_fn(...) if let Expr::Call(call) = arg { - if self.is_task_fn_call(&Expr::Call(call.clone())) { + if self.is_task_fn_call(arg) { let (name, script) = self .extract_step_info_from_task_call(call) .unwrap_or(("unknown".into(), "unknown".into())); @@ -553,15 +536,33 @@ impl WacWalker { } fn walk_if(&mut self, if_stmt: &StmtIf) -> Option<(String, String)> { - let has_steps_in_body = self.body_contains_step(&if_stmt.body); - let has_steps_in_else = self.body_contains_step(&if_stmt.orelse); + self.walk_if_chain( + if_stmt.start(), + &if_stmt.test, + &if_stmt.body, + &if_stmt.elif_else_clauses, + ) + } + + /// An `elif` is laid out as an `if` nested alone in the `else` of the clause before it. + fn walk_if_chain( + &mut self, + start: TextSize, + test: &Expr, + body: &[Stmt], + clauses: &[ElifElseClause], + ) -> Option<(String, String)> { + let has_steps_in_body = self.body_contains_step(body); + let has_steps_in_else = clauses + .iter() + .any(|clause| self.body_contains_step(&clause.body)); if !has_steps_in_body && !has_steps_in_else { return None; } - let line = self.line_index.line_of(if_stmt.range.start().to_usize()); - let condition_source = Self::expr_to_source(&if_stmt.test); + let line = self.line_index.line_of(start.to_usize()); + let condition_source = Self::expr_to_source(test); let branch_id = self.next_id(); let branch_node_id = self.add_node(DagNode { @@ -573,15 +574,19 @@ impl WacWalker { let mut last_ids = Vec::new(); - if let Some((true_first, true_last)) = self.walk_body(&if_stmt.body) { + if let Some((true_first, true_last)) = self.walk_body(body) { self.add_edge(&branch_node_id, &true_first, Some("true".to_string())); last_ids.push(true_last); } else { last_ids.push(branch_node_id.clone()); } - if !if_stmt.orelse.is_empty() { - if let Some((else_first, else_last)) = self.walk_body(&if_stmt.orelse) { + if let Some((clause, clauses)) = clauses.split_first() { + let orelse = match &clause.test { + Some(test) => self.walk_if_chain(clause.start(), test, &clause.body, clauses), + None => self.walk_body(&clause.body), + }; + if let Some((else_first, else_last)) = orelse { self.add_edge(&branch_node_id, &else_first, Some("false".to_string())); last_ids.push(else_last); } else { @@ -611,7 +616,7 @@ impl WacWalker { return None; } - let line = self.line_index.line_of(for_stmt.range.start().to_usize()); + let line = self.line_index.line_of(for_stmt.start().to_usize()); let iter_source = Self::expr_to_source(&for_stmt.iter); let start_id = self.next_id(); @@ -644,7 +649,7 @@ impl WacWalker { return None; } - let line = self.line_index.line_of(while_stmt.range.start().to_usize()); + let line = self.line_index.line_of(while_stmt.start().to_usize()); let condition = Self::expr_to_source(&while_stmt.test); let start_id = self.next_id(); @@ -677,39 +682,14 @@ impl WacWalker { || self.body_contains_step(&try_stmt.orelse) || self.body_contains_step(&try_stmt.finalbody) || try_stmt.handlers.iter().any(|h| match h { - rustpython_parser::ast::ExceptHandler::ExceptHandler(eh) => { - self.body_contains_step(&eh.body) - } + ExceptHandler::ExceptHandler(eh) => self.body_contains_step(&eh.body), }); if !has_steps { return None; } - let line = self.line_index.line_of(try_stmt.range.start().to_usize()); - self.emit_try_catch_branch( - &try_stmt.body, - &try_stmt.handlers, - &try_stmt.finalbody, - line, - ) - } - - fn walk_try_star(&mut self, try_stmt: &StmtTryStar) -> Option<(String, String)> { - let has_steps = self.body_contains_step(&try_stmt.body) - || self.body_contains_step(&try_stmt.orelse) - || self.body_contains_step(&try_stmt.finalbody) - || try_stmt.handlers.iter().any(|h| match h { - rustpython_parser::ast::ExceptHandler::ExceptHandler(eh) => { - self.body_contains_step(&eh.body) - } - }); - - if !has_steps { - return None; - } - - let line = self.line_index.line_of(try_stmt.range.start().to_usize()); + let line = self.line_index.line_of(try_stmt.start().to_usize()); self.emit_try_catch_branch( &try_stmt.body, &try_stmt.handlers, @@ -721,7 +701,7 @@ impl WacWalker { fn emit_try_catch_branch( &mut self, try_body: &[Stmt], - handlers: &[rustpython_parser::ast::ExceptHandler], + handlers: &[ExceptHandler], finally_body: &[Stmt], line: usize, ) -> Option<(String, String)> { @@ -746,7 +726,7 @@ impl WacWalker { // Except handlers for handler in handlers { match handler { - rustpython_parser::ast::ExceptHandler::ExceptHandler(eh) => { + ExceptHandler::ExceptHandler(eh) => { if let Some((catch_first, catch_last)) = self.walk_body(&eh.body) { self.add_edge(&branch_node_id, &catch_first, Some("except".to_string())); last_ids.push(catch_last); @@ -783,7 +763,7 @@ impl WacWalker { } fn walk_return(&mut self, ret: &StmtReturn) -> Option<(String, String)> { - let line = self.line_index.line_of(ret.range.start().to_usize()); + let line = self.line_index.line_of(ret.start().to_usize()); let id = self.next_id(); let node_id = self.add_node(DagNode { id: id.clone(), @@ -796,12 +776,12 @@ impl WacWalker { } /// Extract workflow function parameters (no longer skips ctx) -fn extract_params(args: &rustpython_parser::ast::Arguments) -> Vec { +fn extract_params(args: &Parameters) -> Vec { let mut params = Vec::new(); for arg_with_default in args.args.iter().chain(args.posonlyargs.iter()) { - let name = arg_with_default.def.arg.to_string(); + let name = arg_with_default.parameter.name.to_string(); let typ = arg_with_default - .def + .parameter .annotation .as_ref() .map(|ann| WacWalker::expr_to_source(ann)); @@ -811,59 +791,34 @@ fn extract_params(args: &rustpython_parser::ast::Arguments) -> Vec { } pub fn parse_python_workflow(code: &str) -> Result> { - let ast = rustpython_parser::ast::Suite::parse(code, "") - .map_err(|e| vec![CompileError { message: format!("Parse error: {e}"), line: 0 }])?; + let ast = parse_module(code) + .map_err(|e| vec![CompileError { message: format!("Parse error: {e}"), line: 0 }])? + .into_suite(); // First pass: collect @task functions let task_functions = collect_task_functions(&ast); - // Find the @workflow async def - let workflow_fn = ast.iter().find_map(|stmt| { - if let Stmt::AsyncFunctionDef(func) = stmt { - let has_workflow_decorator = func.decorator_list.iter().any(|dec| { - if let Expr::Name(ExprName { id, .. }) = dec { - id.as_str() == "workflow" - } else { - false - } - }); - if has_workflow_decorator { - return Some(func); + let workflow_fns = || { + ast.iter().filter_map(|stmt| match stmt { + Stmt::FunctionDef(func) + if func.decorator_list.iter().any(|dec| { + matches!(&dec.expression, Expr::Name(ExprName { id, .. }) if id.as_str() == "workflow") + }) => + { + Some(func) } - } - // Also check non-async for error reporting - if let Stmt::FunctionDef(func) = stmt { - let has_workflow_decorator = func.decorator_list.iter().any(|dec| { - if let Expr::Name(ExprName { id, .. }) = dec { - id.as_str() == "workflow" - } else { - false - } - }); - if has_workflow_decorator { - return None; // Will be reported as not-async below - } - } - None - }); + _ => None, + }) + }; - // Check for non-async workflow function - let non_async_workflow = ast.iter().find_map(|stmt| { - if let Stmt::FunctionDef(func) = stmt { - let has_workflow_decorator = func.decorator_list.iter().any(|dec| { - if let Expr::Name(ExprName { id, .. }) = dec { - id.as_str() == "workflow" - } else { - false - } - }); - if has_workflow_decorator { - let line_index = LineIndex::new(code); - return Some(line_index.line_of(func.range.start().to_usize())); - } - } - None - }); + // Find the @workflow async def + let workflow_fn = workflow_fns().find(|func| func.is_async); + + // Check for non-async workflow function. The statement starts at its first decorator, + // the name sits on the `def` line the error points at. + let non_async_workflow = workflow_fns() + .find(|func| !func.is_async) + .map(|func| LineIndex::new(code).line_of(func.name.start().to_usize())); if let Some(line) = non_async_workflow { if workflow_fn.is_none() { @@ -875,7 +830,7 @@ pub fn parse_python_workflow(code: &str) -> Result String { hasher.update(code.as_bytes()); format!("{:x}", hasher.finalize()) } - -trait ToUsize { - fn to_usize(self) -> usize; -} - -impl ToUsize for rustpython_parser::text_size::TextSize { - fn to_usize(self) -> usize { - u32::from(self) as usize - } -} diff --git a/backend/parsers/windmill-parser-wac/tests/python_tests.rs b/backend/parsers/windmill-parser-wac/tests/python_tests.rs index cf117da9e7..c0de489153 100644 --- a/backend/parsers/windmill-parser-wac/tests/python_tests.rs +++ b/backend/parsers/windmill-parser-wac/tests/python_tests.rs @@ -363,3 +363,77 @@ async def main(x: str): )); assert!(matches!(dag.nodes[5].node_type, DagNodeType::Return)); } + +#[test] +fn test_elif_chain_nests_branches() { + let code = r#" +from wmill import workflow, task + +@task +async def a(): ... +@task +async def b(): ... +@task +async def c(): ... + +@workflow +async def main(x: int): + if x > 2: + await a() + elif x == 2: + await b() + else: + await c() +"#; + + let dag = parse_python_workflow(code).expect("should parse"); + let branches: Vec<(&str, usize)> = dag + .nodes + .iter() + .filter_map(|n| match &n.node_type { + DagNodeType::Branch { condition_source } => Some((condition_source.as_str(), n.line)), + _ => None, + }) + .collect(); + assert_eq!(branches, vec![("x > 2", 13), ("x == 2", 15)]); + + // The `elif` branch hangs off the `false` edge of the `if` before it + let (outer, inner) = (&dag.nodes[0].id, &dag.nodes[2].id); + assert!(dag + .edges + .iter() + .any(|e| &e.from == outer && &e.to == inner && e.label.as_deref() == Some("false"))); + assert_eq!( + dag.nodes + .iter() + .filter(|n| matches!(n.node_type, DagNodeType::Merge)) + .count(), + 2 + ); +} + +#[test] +fn test_workflow_with_recent_syntax() { + let code = r#" +from wmill import workflow, task +lazy import polars + +type Rows = list[dict] + +@task +async def load[T](rows: T): ... + +@workflow +async def main(name: str): + try: + await load(t"hello {name}") + except ValueError, TypeError: + pass +"#; + + let dag = parse_python_workflow(code).expect("should parse"); + assert!(dag + .nodes + .iter() + .any(|n| matches!(&n.node_type, DagNodeType::Step { name, .. } if name == "load"))); +} diff --git a/backend/parsers/windmill-parser-wasm/Cargo.lock b/backend/parsers/windmill-parser-wasm/Cargo.lock index 62c0cc5a8d..75c10528df 100644 --- a/backend/parsers/windmill-parser-wasm/Cargo.lock +++ b/backend/parsers/windmill-parser-wasm/Cargo.lock @@ -254,6 +254,35 @@ version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" +[[package]] +name = "attribute-derive" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2f9ef7a68684657ce9da341c934c73b63df7a3585a61e366d84c84ed5d3d167" +dependencies = [ + "attribute-derive-macro", + "manyhow", + "proc-macro2", + "quote", + "syn 3.0.7", +] + +[[package]] +name = "attribute-derive-macro" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300d426e5e73549125bb8e4649bdd5508f82c94ddf898fb0517246e71dfb8e98" +dependencies = [ + "collection_literals", + "interpolator", + "manyhow", + "proc-macro-utils", + "proc-macro2", + "quote", + "quote-use", + "syn 3.0.7", +] + [[package]] name = "autocfg" version = "1.5.0" @@ -428,7 +457,7 @@ dependencies = [ "proc-macro2", "quote", "regex", - "rustc-hash 2.1.1", + "rustc-hash", "shlex", "syn 2.0.117", ] @@ -620,6 +649,15 @@ version = "1.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2e93abca9e28e0a1b9877922aacb20576e05d4679ffa78c3d6dc22a26a216659" +[[package]] +name = "castaway" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dec551ab6e7578819132c713a93c022a05d60159dc86e7a7050223577484c55a" +dependencies = [ + "rustversion", +] + [[package]] name = "cc" version = "1.2.56" @@ -670,6 +708,18 @@ dependencies = [ "rand_core 0.10.1", ] +[[package]] +name = "char_str" +version = "0.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9cba78a5abb515610e10af297615434b019755b5d7be14e559ff5340dff50c4" +dependencies = [ + "castaway", + "get-size2", + "itoa", + "ryu", +] + [[package]] name = "chrono" version = "0.4.44" @@ -779,6 +829,12 @@ version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c9ea0ac24bc397ab3c98583a3c9ba74fa56b09a4449bbe172b9b1ddb016027a" +[[package]] +name = "collection_literals" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2550f75b8cfac212855f6b1885455df8eaee8fe8e246b647d69146142e016084" + [[package]] name = "colorchoice" version = "1.0.4" @@ -795,6 +851,19 @@ dependencies = [ "memchr", ] +[[package]] +name = "compact_str" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79fcda08c33bb58b97008b2cdada6622500e949e060f5913361763121abd2416" +dependencies = [ + "castaway", + "cfg-if", + "itoa", + "static_assertions", + "zmij", +] + [[package]] name = "compression-codecs" version = "0.4.37" @@ -1019,12 +1088,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "crunchy" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" - [[package]] name = "crypto-common" version = "0.1.7" @@ -1093,27 +1156,6 @@ dependencies = [ "powerfmt", ] -[[package]] -name = "derive_more" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a9b99b9cbbe49445b21764dc0625032a89b145a2642e67603e1c936f5458d05" -dependencies = [ - "derive_more-impl", -] - -[[package]] -name = "derive_more-impl" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb7330aeadfbe296029522e6c40f315320aba36fc43a5b3632f3795348f3bd22" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.117", - "unicode-xid", -] - [[package]] name = "des" version = "0.7.0" @@ -1196,6 +1238,12 @@ version = "0.15.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b" +[[package]] +name = "drop_bomb" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9bda8e21c04aca2ae33ffc2fd8c23134f3cac46db123ba97bd9d3f3b8a4a85e1" + [[package]] name = "dunce" version = "1.0.5" @@ -1502,6 +1550,31 @@ dependencies = [ "version_check", ] +[[package]] +name = "get-size-derive2" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28f0daa179069133df60523e0f59d0a6bd2d3db9ccae01b630896d269c4a802e" +dependencies = [ + "attribute-derive", + "quote", + "syn 3.0.7", +] + +[[package]] +name = "get-size2" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "050a5d07359ebfac7f0b74916d94cfa30575e45fdc7e9bfbb0f76d5d4203325a" +dependencies = [ + "compact_str", + "get-size-derive2", + "hashbrown 0.17.1", + "ordermap", + "smallvec", + "thin-vec", +] + [[package]] name = "gethostname" version = "0.4.3" @@ -1686,6 +1759,12 @@ version = "0.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + [[package]] name = "hashlink" version = "0.10.0" @@ -1758,7 +1837,7 @@ dependencies = [ "hashbrown 0.14.5", "new_debug_unreachable", "once_cell", - "rustc-hash 2.1.1", + "rustc-hash", "triomphe", ] @@ -1870,7 +1949,7 @@ dependencies = [ "libc", "percent-encoding", "pin-project-lite", - "socket2 0.6.3", + "socket2", "system-configuration", "tokio", "tower-service", @@ -2012,16 +2091,22 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.13.0" +version = "2.14.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7714e70437a7dc3ac8eb7e6f8df75fd8eb422675fc7678aff7364301092b1017" +checksum = "cc4e190f5d26ca7051642629da2c52fc03bde85a03197c99408dcd291734c855" dependencies = [ "equivalent", - "hashbrown 0.16.1", + "hashbrown 0.17.1", "serde", "serde_core", ] +[[package]] +name = "interpolator" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71dd52191aae121e8611f1e8dc3e324dd0dd1dee1e6dd91d10ee07a3cfb4d9d8" + [[package]] name = "inventory" version = "0.3.22" @@ -2082,15 +2167,6 @@ version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" -[[package]] -name = "itertools" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1c173a5686ce8bfa551b3563d0c2170bf24ca44da99c7ca4bfdab5418c3fe57" -dependencies = [ - "either", -] - [[package]] name = "itertools" version = "0.13.0" @@ -2109,6 +2185,15 @@ dependencies = [ "either", ] +[[package]] +name = "itertools" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b4baf93f58d4425749ca49a51c50ebab072c5df6994d08fed93541c331481dc" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.17" @@ -2212,12 +2297,6 @@ version = "0.2.19" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a4933f3f57a8e9d9da04db23fb153356ecaf00cbd14aee46279c33dc80925c37" -[[package]] -name = "lalrpop-util" -version = "0.20.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "507460a910eb7b32ee961886ff48539633b788a36b65692b95f225b844c82553" - [[package]] name = "lazy_static" version = "1.5.0" @@ -2388,61 +2467,26 @@ dependencies = [ ] [[package]] -name = "malachite" -version = "0.4.22" +name = "manyhow" +version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2fbdf9cb251732db30a7200ebb6ae5d22fe8e11397364416617d2c2cf0c51cb5" +checksum = "8c69ea6e469ca785c846b25dd69eac9e9c916b51d8ada1f212866787ee1da12d" dependencies = [ - "malachite-base", - "malachite-nz", - "malachite-q", + "manyhow-macros", + "proc-macro2", + "quote", + "syn 3.0.7", ] [[package]] -name = "malachite-base" -version = "0.4.22" +name = "manyhow-macros" +version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ea0ed76adf7defc1a92240b5c36d5368cfe9251640dcce5bd2d0b7c1fd87aeb" +checksum = "22fc5c1b2c4000056dae3619fea36e79ce79dabfee4907d2dcad6612b189e370" dependencies = [ - "hashbrown 0.14.5", - "itertools 0.11.0", - "libm", - "ryu", -] - -[[package]] -name = "malachite-bigint" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d149aaa2965d70381709d9df4c7ee1fc0de1c614a4efc2ee356f5e43d68749f8" -dependencies = [ - "derive_more", - "malachite", - "num-integer", - "num-traits", - "paste", -] - -[[package]] -name = "malachite-nz" -version = "0.4.22" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34a79feebb2bc9aa7762047c8e5495269a367da6b5a90a99882a0aeeac1841f7" -dependencies = [ - "itertools 0.11.0", - "libm", - "malachite-base", -] - -[[package]] -name = "malachite-q" -version = "0.4.22" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50f235d5747b1256b47620f5640c2a17a88c7569eebdf27cd9cb130e1a619191" -dependencies = [ - "itertools 0.11.0", - "malachite-base", - "malachite-nz", + "proc-macro-utils", + "proc-macro2", + "quote", ] [[package]] @@ -2949,6 +2993,15 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" +[[package]] +name = "ordermap" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16a756ddcdeb6c0efc8927bdf0ed112885815a35ba350472a467be5104f01ca1" +dependencies = [ + "indexmap", +] + [[package]] name = "owo-colors" version = "4.3.0" @@ -2984,12 +3037,6 @@ dependencies = [ "windows-link", ] -[[package]] -name = "paste" -version = "1.0.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" - [[package]] name = "pem" version = "3.0.6" @@ -3286,6 +3333,17 @@ dependencies = [ "version_check", ] +[[package]] +name = "proc-macro-utils" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eeaf08a13de400bc215877b5bdc088f241b12eb42f0a548d3390dc1c56bb7071" +dependencies = [ + "proc-macro2", + "quote", + "smallvec", +] + [[package]] name = "proc-macro2" version = "1.0.106" @@ -3401,9 +3459,9 @@ dependencies = [ "pin-project-lite", "quinn-proto", "quinn-udp", - "rustc-hash 2.1.1", + "rustc-hash", "rustls", - "socket2 0.6.3", + "socket2", "thiserror 2.0.18", "tokio", "tracing", @@ -3422,7 +3480,7 @@ dependencies = [ "lru-slab", "rand 0.9.5", "ring", - "rustc-hash 2.1.1", + "rustc-hash", "rustls", "rustls-pki-types", "slab", @@ -3441,7 +3499,7 @@ dependencies = [ "cfg_aliases", "libc", "once_cell", - "socket2 0.6.3", + "socket2", "tracing", "windows-sys 0.60.2", ] @@ -3455,6 +3513,28 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "quote-use" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48356e493823dc3c1d7e9fc7b3ff204ecec48afbdd718fc4134091b88eb63f3d" +dependencies = [ + "quote", + "quote-use-macros", +] + +[[package]] +name = "quote-use-macros" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8cf1c5de6a7cc231c71e03c77aa4c7019097859ea7029156b54d2894a8430704" +dependencies = [ + "proc-macro-utils", + "proc-macro2", + "quote", + "syn 3.0.7", +] + [[package]] name = "r-efi" version = "5.3.0" @@ -3835,6 +3915,106 @@ dependencies = [ "zeroize", ] +[[package]] +name = "ruff_python_ast" +version = "0.0.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15434b06df7d930fad2818639a4e0fec4af9c770121c45a96b72da7feaae9ead" +dependencies = [ + "aho-corasick", + "arrayvec", + "bitflags", + "char_str", + "compact_str", + "get-size2", + "is-macro", + "memchr", + "ruff_python_trivia", + "ruff_source_file", + "ruff_text_size", + "rustc-hash", + "thin-vec", + "thiserror 2.0.18", +] + +[[package]] +name = "ruff_python_parser" +version = "0.0.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37a22811d752aa9e255d7dd72524d14352cf9bc654ef79b399bf1eeeb6640fbb" +dependencies = [ + "bitflags", + "bstr", + "drop_bomb", + "get-size2", + "hashbrown 0.17.1", + "memchr", + "ruff_python_ast", + "ruff_python_trivia", + "ruff_text_size", + "ruff_unicode_names2", + "rustc-hash", + "stacker", + "static_assertions", + "thin-vec", + "unicode-ident", + "unicode-normalization", +] + +[[package]] +name = "ruff_python_trivia" +version = "0.0.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0322a5059fe0feb2e010ca6f31c24ac289f6b7e9e38359539491b10c57f1c5bc" +dependencies = [ + "itertools 0.15.0", + "memchr", + "ruff_source_file", + "ruff_text_size", + "rustc-hash", + "unicode-ident", + "unicode-normalization", +] + +[[package]] +name = "ruff_source_file" +version = "0.0.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6ae809a289b5c791f82aa45df33f2a92323e7a1c1d265c10e9ac884027a2e43" +dependencies = [ + "memchr", + "ruff_text_size", +] + +[[package]] +name = "ruff_text_size" +version = "0.0.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "daf6344d6915ced9316312c1ee3f24c4300c54295bc4c3b8f4e95b633ae5443b" +dependencies = [ + "get-size2", +] + +[[package]] +name = "ruff_unicode_names2" +version = "0.0.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13418b8fd029d597e35dedc36e91f47122d26075e4c86557574d9441753fb375" +dependencies = [ + "phf 0.11.3", + "ruff_unicode_names2_generator", +] + +[[package]] +name = "ruff_unicode_names2_generator" +version = "0.0.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37fd82af49d5685d9054e68185987acaaf62542f671b6df8935155664876bc39" +dependencies = [ + "phf_codegen", + "rand 0.8.6", +] + [[package]] name = "rust_decimal" version = "1.40.0" @@ -3857,12 +4037,6 @@ version = "0.1.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b50b8869d9fc858ce7266cce0194bd74df58b9d0e3f6df3a9fc8eb470d95c09d" -[[package]] -name = "rustc-hash" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" - [[package]] name = "rustc-hash" version = "2.1.1" @@ -3971,63 +4145,6 @@ dependencies = [ "untrusted 0.9.0", ] -[[package]] -name = "rustpython-ast" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4cdaf8ee5c1473b993b398c174641d3aa9da847af36e8d5eb8291930b72f31a5" -dependencies = [ - "is-macro", - "malachite-bigint", - "rustpython-parser-core", - "static_assertions", -] - -[[package]] -name = "rustpython-parser" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "868f724daac0caf9bd36d38caf45819905193a901e8f1c983345a68e18fb2abb" -dependencies = [ - "anyhow", - "is-macro", - "itertools 0.11.0", - "lalrpop-util", - "log", - "malachite-bigint", - "num-traits", - "phf 0.11.3", - "phf_codegen", - "rustc-hash 1.1.0", - "rustpython-ast", - "rustpython-parser-core", - "tiny-keccak", - "unic-emoji-char", - "unic-ucd-ident", - "unicode_names2", -] - -[[package]] -name = "rustpython-parser-core" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4b6c12fa273825edc7bccd9a734f0ad5ba4b8a2f4da5ff7efe946f066d0f4ad" -dependencies = [ - "is-macro", - "memchr", - "rustpython-parser-vendored", -] - -[[package]] -name = "rustpython-parser-vendored" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04fcea49a4630a3a5d940f4d514dc4f575ed63c14c3e3ed07146634aed7f67a6" -dependencies = [ - "memchr", - "once_cell", -] - [[package]] name = "rustversion" version = "1.0.22" @@ -4040,6 +4157,12 @@ version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" +[[package]] +name = "ryu-js" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04d056b875a9d2e6cb9a61d127afee9ac5999b9f87bcb32079d1318e505be714" + [[package]] name = "same-file" version = "1.0.6" @@ -4409,16 +4532,6 @@ dependencies = [ "version_check", ] -[[package]] -name = "socket2" -version = "0.5.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e22376abed350d73dd1cd119b57ffccad95b4e585a7cda43e286245ce23c0678" -dependencies = [ - "libc", - "windows-sys 0.52.0", -] - [[package]] name = "socket2" version = "0.6.3" @@ -4680,15 +4793,15 @@ checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" [[package]] name = "stacker" -version = "0.1.23" +version = "0.1.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08d74a23609d509411d10e2176dc2a4346e3b4aea2e7b1869f19fdedbc71c013" +checksum = "707f49d46706bacf8a2b00d51dace3f9de527c13eec3778f570c411f89e69967" dependencies = [ "cc", "cfg-if", "libc", "psm", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -4830,7 +4943,7 @@ dependencies = [ "new_debug_unreachable", "num-bigint", "once_cell", - "rustc-hash 2.1.1", + "rustc-hash", "serde", "siphasher 0.3.11", "swc_atoms", @@ -4852,7 +4965,7 @@ dependencies = [ "num-bigint", "once_cell", "phf 0.11.3", - "rustc-hash 2.1.1", + "rustc-hash", "string_enum", "swc_atoms", "swc_common", @@ -4871,7 +4984,7 @@ dependencies = [ "either", "num-bigint", "phf 0.11.3", - "rustc-hash 2.1.1", + "rustc-hash", "seq-macro", "serde", "smallvec", @@ -4974,6 +5087,17 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "syn" +version = "3.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d62a2e0561533f2ca2561d0cf27fd9fedb640a1bf2616ff5d5c80d99017faadc" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + [[package]] name = "sync_wrapper" version = "1.0.2" @@ -5102,6 +5226,12 @@ dependencies = [ "unicode-width 0.2.2", ] +[[package]] +name = "thin-vec" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6a4b9ba8738cb4a4f399d37e266becfd475e75eb73425b87a05a2f2039ba63e" + [[package]] name = "thiserror" version = "1.0.69" @@ -5193,15 +5323,6 @@ dependencies = [ "time-core", ] -[[package]] -name = "tiny-keccak" -version = "2.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c9d3793400a45f954c52e73d068316d76b6f4e36977e3fcebb13a2721e80237" -dependencies = [ - "crunchy", -] - [[package]] name = "tinystr" version = "0.8.2" @@ -5229,9 +5350,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.46.1" +version = "1.47.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0cc3a2344dafbe23a245241fe8b09735b521110d30fcefbbd5feb1797ca35d17" +checksum = "89e49afdadebb872d3145a5638b59eb0691ea23e46ca484037cfab3b76b95038" dependencies = [ "backtrace", "bytes", @@ -5242,10 +5363,10 @@ dependencies = [ "pin-project-lite", "signal-hook-registry", "slab", - "socket2 0.5.10", + "socket2", "tokio-macros", "tracing", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -5289,7 +5410,7 @@ dependencies = [ "postgres-protocol", "postgres-types", "rand 0.10.1", - "socket2 0.6.3", + "socket2", "tokio", "tokio-util", "whoami 2.1.0", @@ -5325,6 +5446,7 @@ dependencies = [ "bytes", "futures-core", "futures-sink", + "futures-util", "pin-project-lite", "tokio", ] @@ -5663,17 +5785,6 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "80d7ff825a6a654ee85a63e80f92f054f904f21e7d12da4e22f9834a4aaa35bc" -[[package]] -name = "unic-emoji-char" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b07221e68897210270a38bde4babb655869637af0f69407f96053a34f76494d" -dependencies = [ - "unic-char-property", - "unic-char-range", - "unic-ucd-version", -] - [[package]] name = "unic-ucd-category" version = "0.9.0" @@ -5686,17 +5797,6 @@ dependencies = [ "unic-ucd-version", ] -[[package]] -name = "unic-ucd-ident" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e230a37c0381caa9219d67cf063aa3a375ffed5bf541a452db16e744bdab6987" -dependencies = [ - "unic-char-property", - "unic-char-range", - "unic-ucd-version", -] - [[package]] name = "unic-ucd-version" version = "0.9.0" @@ -5775,28 +5875,6 @@ version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" -[[package]] -name = "unicode_names2" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1673eca9782c84de5f81b82e4109dcfb3611c8ba0d52930ec4a9478f547b2dd" -dependencies = [ - "phf 0.11.3", - "unicode_names2_generator", -] - -[[package]] -name = "unicode_names2_generator" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b91e5b84611016120197efd7dc93ef76774f4e084cd73c9fb3ea4a86c570c56e" -dependencies = [ - "getopts", - "log", - "phf_codegen", - "rand 0.8.6", -] - [[package]] name = "unscanny" version = "0.1.0" @@ -6216,7 +6294,7 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "windmill-common" -version = "1.821.0" +version = "1.830.0" dependencies = [ "aho-corasick", "anyhow", @@ -6260,12 +6338,14 @@ dependencies = [ "pin-project-lite", "pkcs1", "postgres-native-tls", + "postgres-protocol", "quick_cache", "rand 0.9.5", "regex", "reqwest", "reqwest-middleware", "reqwest-retry", + "ryu-js", "semver", "serde", "serde_json", @@ -6302,7 +6382,7 @@ dependencies = [ [[package]] name = "windmill-macros" -version = "1.821.0" +version = "1.830.0" dependencies = [ "proc-macro2", "quote", @@ -6314,7 +6394,7 @@ dependencies = [ [[package]] name = "windmill-parser" -version = "1.821.0" +version = "1.830.0" dependencies = [ "convert_case", "serde", @@ -6323,7 +6403,7 @@ dependencies = [ [[package]] name = "windmill-parser-bash" -version = "1.821.0" +version = "1.830.0" dependencies = [ "anyhow", "lazy_static", @@ -6335,7 +6415,7 @@ dependencies = [ [[package]] name = "windmill-parser-csharp" -version = "1.821.0" +version = "1.830.0" dependencies = [ "anyhow", "serde_json", @@ -6347,7 +6427,7 @@ dependencies = [ [[package]] name = "windmill-parser-go" -version = "1.821.0" +version = "1.830.0" dependencies = [ "anyhow", "gosyn", @@ -6359,7 +6439,7 @@ dependencies = [ [[package]] name = "windmill-parser-graphql" -version = "1.821.0" +version = "1.830.0" dependencies = [ "anyhow", "lazy_static", @@ -6371,7 +6451,7 @@ dependencies = [ [[package]] name = "windmill-parser-java" -version = "1.821.0" +version = "1.830.0" dependencies = [ "anyhow", "serde_json", @@ -6383,7 +6463,7 @@ dependencies = [ [[package]] name = "windmill-parser-nu" -version = "1.821.0" +version = "1.830.0" dependencies = [ "anyhow", "nu-parser", @@ -6394,7 +6474,7 @@ dependencies = [ [[package]] name = "windmill-parser-php" -version = "1.821.0" +version = "1.830.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -6405,30 +6485,30 @@ dependencies = [ [[package]] name = "windmill-parser-py" -version = "1.821.0" +version = "1.830.0" dependencies = [ "anyhow", "itertools 0.14.0", - "rustpython-ast", - "rustpython-parser", + "ruff_python_ast", + "ruff_python_parser", "serde_json", "windmill-parser", ] [[package]] name = "windmill-parser-py-asset" -version = "1.821.0" +version = "1.830.0" dependencies = [ "anyhow", - "rustpython-ast", - "rustpython-parser", + "ruff_python_ast", + "ruff_python_parser", "windmill-parser", "windmill-parser-sql-asset", ] [[package]] name = "windmill-parser-py-imports" -version = "1.821.0" +version = "1.830.0" dependencies = [ "anyhow", "async-recursion", @@ -6438,7 +6518,8 @@ dependencies = [ "phf 0.11.3", "regex", "regex-lite", - "rustpython-parser", + "ruff_python_ast", + "ruff_python_parser", "serde", "serde_json", "sqlx", @@ -6450,7 +6531,7 @@ dependencies = [ [[package]] name = "windmill-parser-r" -version = "1.821.0" +version = "1.830.0" dependencies = [ "anyhow", "serde_json", @@ -6462,7 +6543,7 @@ dependencies = [ [[package]] name = "windmill-parser-ruby" -version = "1.821.0" +version = "1.830.0" dependencies = [ "anyhow", "lazy_static", @@ -6476,7 +6557,7 @@ dependencies = [ [[package]] name = "windmill-parser-rust" -version = "1.821.0" +version = "1.830.0" dependencies = [ "anyhow", "convert_case", @@ -6493,7 +6574,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql" -version = "1.821.0" +version = "1.830.0" dependencies = [ "anyhow", "lazy_static", @@ -6506,7 +6587,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql-asset" -version = "1.821.0" +version = "1.830.0" dependencies = [ "anyhow", "serde", @@ -6518,7 +6599,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts" -version = "1.821.0" +version = "1.830.0" dependencies = [ "anyhow", "lazy_static", @@ -6536,7 +6617,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts-asset" -version = "1.821.0" +version = "1.830.0" dependencies = [ "anyhow", "serde-wasm-bindgen", @@ -6552,11 +6633,12 @@ dependencies = [ [[package]] name = "windmill-parser-wac" -version = "1.821.0" +version = "1.830.0" dependencies = [ "anyhow", - "rustpython-ast", - "rustpython-parser", + "ruff_python_ast", + "ruff_python_parser", + "ruff_text_size", "serde", "serde_json", "sha2 0.10.9", @@ -6568,7 +6650,7 @@ dependencies = [ [[package]] name = "windmill-parser-wasm" -version = "1.821.0" +version = "1.830.0" dependencies = [ "anyhow", "getrandom 0.2.17", @@ -6600,7 +6682,7 @@ dependencies = [ [[package]] name = "windmill-parser-yaml" -version = "1.821.0" +version = "1.830.0" dependencies = [ "anyhow", "lazy_static", @@ -6614,7 +6696,7 @@ dependencies = [ [[package]] name = "windmill-types" -version = "1.821.0" +version = "1.830.0" dependencies = [ "anyhow", "bitflags", diff --git a/backend/parsers/windmill-parser-wasm/Cargo.toml b/backend/parsers/windmill-parser-wasm/Cargo.toml index 387a98aa1f..8aebf20597 100644 --- a/backend/parsers/windmill-parser-wasm/Cargo.toml +++ b/backend/parsers/windmill-parser-wasm/Cargo.toml @@ -12,7 +12,7 @@ resolver = "2" members = ["."] [workspace.package] -version = "1.821.0" +version = "1.830.0" edition = "2021" authors = ["Ruben Fiszel "] diff --git a/backend/src/main.rs b/backend/src/main.rs index 23978d7e62..fd953213a4 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -44,32 +44,33 @@ use windmill_common::{ CRITICAL_ALERTS_ON_TOKEN_EXPIRY_SETTING, CRITICAL_ALERT_MUTE_UI_SETTING, CRITICAL_ALERT_MUTE_ZOMBIE_JOB_RESTART_SETTING, CRITICAL_ERROR_CHANNELS_SETTING, CUSTOM_TAGS_SETTING, DEFAULT_TAGS_PER_WORKSPACE_SETTING, DEFAULT_TAGS_WORKSPACES_SETTING, - DISABLE_PASSWORD_LOGIN_SETTING, EMAIL_DOMAIN_SETTING, ENV_SETTINGS, - EXPOSE_DEBUG_METRICS_SETTING, EXPOSE_METRICS_SETTING, EXTRA_PIP_INDEX_URL_SETTING, - FORK_WORKSPACE_TAG_APPEND_FORK_SUFFIX_SETTING, + DEPENDENCY_JOB_TAG_SETTING, DISABLE_PASSWORD_LOGIN_SETTING, EMAIL_DOMAIN_SETTING, + ENV_SETTINGS, EXPOSE_DEBUG_METRICS_SETTING, EXPOSE_METRICS_SETTING, + EXTRA_PIP_INDEX_URL_SETTING, FORK_WORKSPACE_TAG_APPEND_FORK_SUFFIX_SETTING, HTTP_ROUTE_DEFAULT_ALLOWED_ORIGINS_SETTING, HTTP_ROUTE_WORKSPACED_ROUTE_SETTING, HUB_API_SECRET_SETTING, HUB_BASE_URL_SETTING, INDEXER_SETTING, INSTANCE_EVENTS_WEBHOOK_SETTING, INSTANCE_PYTHON_VERSION_SETTING, JOB_DEFAULT_TIMEOUT_SECS_SETTING, JOB_ISOLATION_SETTING, JWT_SECRET_SETTING, KEEP_JOB_DIR_SETTING, LICENSE_KEY_SETTING, MAVEN_REPOS_SETTING, MAVEN_SETTINGS_XML_SETTING, MCP_DISABLE_TOKEN_QUERY_PARAM_SETTING, MONITOR_LOGS_ON_OBJECT_STORE_SETTING, - NO_DEFAULT_MAVEN_SETTING, - NPM_CONFIG_REGISTRY_SETTING, NSJAIL_TMPFS_SIZE_MB_SETTING, NSJAIL_TMP_BACKING_SETTING, - NUGET_CONFIG_SETTING, OAUTH_SETTING, OTEL_SETTING, OTEL_TRACES_RETENTION_SECS_SETTING, - OTEL_TRACING_PROXY_SETTING, PIP_INDEX_URL_SETTING, POWERSHELL_REPO_PAT_SETTING, - POWERSHELL_REPO_URL_SETTING, PREVIEW_TAGS_OVERRIDE_SETTING, REQUEST_SIZE_LIMIT_SETTING, - REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING, RESTART_COORDINATION_SETTING, - RETENTION_PERIOD_SECS_OVERRIDES_SETTING, RETENTION_PERIOD_SECS_SETTING, RUBY_REPOS_SETTING, - SAML_METADATA_SETTING, SANDBOX_IMAGE_CACHE_MAX_MB_SETTING, - SANDBOX_IMAGE_DEFAULT_REGISTRY_SETTING, SANDBOX_IMAGE_MAX_SIZE_MB_SETTING, - SANDBOX_IMAGE_PULL_POLICY_SETTING, SANDBOX_REGISTRY_AUTH_SETTING, SCIM_TOKEN_SETTING, - SERVICE_LOG_RETENTION_SECS_SETTING, SMTP_SETTING, STORE_AUDIT_LOGS_S3_SETTING, + NO_DEFAULT_MAVEN_SETTING, NPM_CONFIG_REGISTRY_SETTING, NSJAIL_TMPFS_SIZE_MB_SETTING, + NSJAIL_TMP_BACKING_SETTING, NUGET_CONFIG_SETTING, OAUTH_SETTING, OTEL_SETTING, + OTEL_TRACES_RETENTION_SECS_SETTING, OTEL_TRACING_PROXY_SETTING, PIP_INDEX_URL_SETTING, + POWERSHELL_REPO_PAT_SETTING, POWERSHELL_REPO_URL_SETTING, PREVIEW_TAGS_OVERRIDE_SETTING, + REQUEST_SIZE_LIMIT_SETTING, REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING, + RESTART_COORDINATION_SETTING, RETENTION_PERIOD_SECS_OVERRIDES_SETTING, + RETENTION_PERIOD_SECS_SETTING, RUBY_REPOS_SETTING, SAML_METADATA_SETTING, + SANDBOX_IMAGE_CACHE_MAX_MB_SETTING, SANDBOX_IMAGE_DEFAULT_REGISTRY_SETTING, + SANDBOX_IMAGE_MAX_SIZE_MB_SETTING, SANDBOX_IMAGE_PULL_POLICY_SETTING, + SANDBOX_REGISTRY_AUTH_SETTING, SCIM_TOKEN_SETTING, SERVICE_LOG_RETENTION_SECS_SETTING, + SMTP_SETTING, STORE_AUDIT_LOGS_S3_SETTING, SYNC_HUB_RESOURCE_TYPES_DAILY_SETTING, TEAMS_SETTING, TIMEOUT_WAIT_RESULT_SETTING, UV_EXCLUDE_NEWER_SETTING, UV_INDEX_STRATEGY_SETTING, UV_PYTHON_INSTALL_MIRROR_SETTING, WORKSPACE_FAIRNESS_DURATION_SECS_SETTING, WORKSPACE_FAIRNESS_ENABLED_SETTING, WORKSPACE_FAIRNESS_MAX_PERCENT_SETTING, WORKSPACE_FAIRNESS_MIN_TOTAL_SETTING, WORKSPACE_MAX_QUEUED_JOBS_SETTING, WORKSPACE_REGISTRIES_SETTING, }, + hub_resource_types, scripts::ScriptLang, stats_oss::schedule_stats, triggers::TriggerKind, @@ -122,28 +123,28 @@ use windmill_worker::{ BUN_BUNDLE_CACHE_DIR, BUN_CACHE_DIR, CSHARP_CACHE_DIR, DENO_CACHE_DIR, DENO_CACHE_DIR_DEPS, DENO_CACHE_DIR_NPM, GO_BIN_CACHE_DIR, GO_CACHE_DIR, JAVA_CACHE_DIR, NU_CACHE_DIR, POWERSHELL_CACHE_DIR, PY310_CACHE_DIR, PY311_CACHE_DIR, PY312_CACHE_DIR, PY313_CACHE_DIR, - RUBY_CACHE_DIR, RUST_CACHE_DIR, R_CACHE_DIR, TAR_JAVA_CACHE_DIR, UV_CACHE_DIR, + PY314_CACHE_DIR, PY315_CACHE_DIR, RUBY_CACHE_DIR, RUST_CACHE_DIR, R_CACHE_DIR, + TAR_JAVA_CACHE_DIR, UV_CACHE_DIR, }; use crate::monitor::{ - initial_load, load_concurrency_key_max_queued, load_disable_password_login, - load_fork_workspace_tag_append_fork_suffix, load_keep_job_dir, - load_mcp_disable_token_query_param, load_metrics_debug_enabled, - load_preview_tags_override, load_require_preexisting_user, load_retention_period_overrides, - load_tag_per_workspace_enabled, load_tag_per_workspace_workspaces, - load_workspace_fairness_duration_secs, load_workspace_fairness_enabled, - load_workspace_fairness_max_percent, load_workspace_fairness_min_total, - load_workspace_max_queued_jobs, monitor_db, reload_app_workspaced_route_setting, - reload_audit_log_retention_days_setting, reload_base_url_setting, - reload_bun_install_min_release_age_setting, reload_bunfig_install_scopes_setting, - reload_critical_alert_mute_ui_setting, reload_critical_alert_mute_zombie_job_restart_setting, + initial_load, load_concurrency_key_max_queued, load_dependency_job_tag, + load_disable_password_login, load_fork_workspace_tag_append_fork_suffix, load_keep_job_dir, + load_mcp_disable_token_query_param, load_metrics_debug_enabled, load_preview_tags_override, + load_require_preexisting_user, load_retention_period_overrides, load_tag_per_workspace_enabled, + load_tag_per_workspace_workspaces, load_workspace_fairness_duration_secs, + load_workspace_fairness_enabled, load_workspace_fairness_max_percent, + load_workspace_fairness_min_total, load_workspace_max_queued_jobs, monitor_db, + reload_app_workspaced_route_setting, reload_audit_log_retention_days_setting, + reload_base_url_setting, reload_bun_install_min_release_age_setting, + reload_bunfig_install_scopes_setting, reload_critical_alert_mute_ui_setting, + reload_critical_alert_mute_zombie_job_restart_setting, reload_critical_alerts_on_token_expiry_setting, reload_critical_error_channels_setting, reload_extra_pip_index_url_setting, reload_http_route_default_allowed_origins_setting, - reload_http_route_workspaced_route_setting, - reload_hub_api_secret_setting, reload_hub_base_url_setting, - reload_instance_events_webhook_setting, reload_job_default_timeout_setting, - reload_job_isolation_setting, reload_jwt_secret_setting, reload_license_key, - reload_npm_config_registry_setting, reload_nsjail_tmp_backing_setting, + reload_http_route_workspaced_route_setting, reload_hub_api_secret_setting, + reload_hub_base_url_setting, reload_instance_events_webhook_setting, + reload_job_default_timeout_setting, reload_job_isolation_setting, reload_jwt_secret_setting, + reload_license_key, reload_npm_config_registry_setting, reload_nsjail_tmp_backing_setting, reload_nsjail_tmpfs_size_setting, reload_otel_traces_retention_secs_setting, reload_otel_tracing_proxy_setting, reload_pip_index_url_setting, reload_retention_period_setting, reload_sandbox_image_cache_max_setting, @@ -403,112 +404,16 @@ async fn cache_hub_scripts(file_path: Option) -> anyhow::Result<()> { Ok(()) } -/// Raw resource type from hub API (schema is a JSON string) -#[derive(serde::Deserialize)] -struct HubResourceTypeRaw { - pub id: i64, - pub name: String, - pub schema: Option, - pub app: String, - pub description: Option, - /// Absent from hubs predating the column, and from caches written before it. - #[serde(default)] - pub format_extension: Option, - /// Doubly optional, so a hub predating the field (no key) is told apart from a type the - /// hub leaves unnamed (null). - #[serde( - default, - deserialize_with = "windmill_common::more_serde::double_option" - )] - pub display_name: Option>, -} - - -/// Processed resource type with parsed schema -#[derive(serde::Deserialize, serde::Serialize, Clone)] -pub struct HubResourceType { - pub id: i64, - pub name: String, - pub schema: Option, - pub app: String, - pub description: Option, - /// Doubly optional on purpose. A cache written before this column has no key at - /// all (`None`) and must leave the stored extension alone; one written since - /// always writes the key, so an explicit null (`Some(None)`) is the hub genuinely - /// dropping it and must clear. A single `Option` conflates the two, and picking - /// either meaning breaks the other — as does plain serde, which folds `null` - /// into the outer `None`, hence the wrapping deserializer. - #[serde( - default, - deserialize_with = "windmill_common::more_serde::double_option", - skip_serializing_if = "Option::is_none" - )] - pub format_extension: Option>, - /// Doubly optional like `format_extension`: a cache written before the field leaves the - /// stored name alone, while a null from the hub clears it. - #[serde( - default, - deserialize_with = "windmill_common::more_serde::double_option", - skip_serializing_if = "Option::is_none" - )] - pub display_name: Option>, -} - -const HUB_RT_CACHE_FILE: &str = "resource_types.json"; - async fn cache_hub_resource_types() -> anyhow::Result<()> { println!("Caching resource types from hub..."); - let response = HTTP_CLIENT - .get(format!("{}/resource_types/list", DEFAULT_HUB_BASE_URL)) - .header("Accept", "application/json") - .send() - .await - .with_context(|| "Failed to fetch resource types from hub")?; - - if !response.status().is_success() { - anyhow::bail!( - "Failed to fetch resource types from hub: {}", - response.status() - ); - } - - let raw_types: Vec = response - .json::>() - .await - .with_context(|| "Failed to parse resource types from hub")?; - - // Parse schema strings into JSON values - let resource_types: Vec = raw_types - .into_iter() - .filter_map(|rt| { - let schema = match rt.schema { - Some(s) => match serde_json::from_str(&s) { - Ok(v) => Some(v), - Err(e) => { - println!("Warning: failed to parse schema for {}: {}", rt.name, e); - return None; - } - }, - None => None, - }; - Some(HubResourceType { - id: rt.id, - name: rt.name, - schema, - app: rt.app, - description: rt.description, - format_extension: Some(rt.format_extension), - display_name: rt.display_name, - }) - }) - .collect(); + let resource_types = hub_resource_types::fetch(DEFAULT_HUB_BASE_URL, None, None).await?; println!("Fetched {} resource types from hub", resource_types.len()); create_dir_all(&*HUB_RT_CACHE_DIR)?; - let cache_path = format!("{}/{}", *HUB_RT_CACHE_DIR, HUB_RT_CACHE_FILE); + let cache_path = hub_resource_types::cache_path(); let content = serde_json::to_string_pretty(&resource_types) .with_context(|| "Failed to serialize resource types")?; @@ -519,147 +424,6 @@ async fn cache_hub_resource_types() -> anyhow::Result<()> { Ok(()) } -pub async fn sync_cached_resource_types(db: &sqlx::Pool) -> anyhow::Result<()> { - let cache_path = format!("{}/{}", *HUB_RT_CACHE_DIR, HUB_RT_CACHE_FILE); - - if tokio::fs::metadata(&cache_path).await.is_err() { - tracing::info!( - "No cached resource types found at {}, skipping sync", - cache_path - ); - return Ok(()); - } - - tracing::info!("Syncing cached resource types to admins workspace..."); - - let content = tokio::fs::read_to_string(&cache_path) - .await - .with_context(|| format!("Failed to read cache file from {}", cache_path))?; - - let cached_types: Vec = - serde_json::from_str(&content).with_context(|| "Failed to parse cached resource types")?; - - tracing::info!("Found {} cached resource types", cached_types.len()); - - // Get existing resource types in admins workspace. `format_extension` is part of - // the comparison below, so a type whose only change is gaining or losing it is - // not mistaken for unchanged; `is_fileset` decides whether it may take one. - let existing_types: Vec<( - String, - Option, - Option, - Option, - bool, - Option, - )> = sqlx::query_as( - "SELECT name, schema, description, format_extension, is_fileset, display_name FROM resource_type WHERE workspace_id = 'admins'", - ) - .fetch_all(db) - .await - .with_context(|| "Failed to fetch existing resource types")?; - - let existing_map: std::collections::HashMap< - String, - ( - Option, - Option, - Option, - bool, - Option, - ), - > = existing_types - .into_iter() - .map( - |(name, schema, desc, format_extension, is_fileset, display_name)| { - ( - name, - (schema, desc, format_extension, is_fileset, display_name), - ) - }, - ) - .collect(); - - let mut synced_count = 0; - let mut skipped_count = 0; - - for rt in cached_types { - let existing = existing_map.get(&rt.name); - let is_fileset = existing.map(|(_, _, _, f, _)| *f).unwrap_or(false); - let stored_extension = existing.and_then(|(_, _, e, _, _)| e.clone()); - let stored_display_name = existing.and_then(|(_, _, _, _, n)| n.clone()); - // A fileset is a set of files, so it cannot also be one file. Create, update - // and the manual sync all reject the pair; this writer would otherwise - // persist it onto a same-named local fileset. - // - // A cache with no key at all leaves the stored value alone, so the target is - // what is already there — which is also what makes the comparison below - // agree with the write instead of re-upserting the row on every boot. - let format_extension = if is_fileset { - None - } else { - match &rt.format_extension { - Some(from_cache) => from_cache.clone(), - None => stored_extension.clone(), - } - }; - // No key in the cache leaves the stored name alone, as for the extension. So does a name - // too long for the column: one bad entry must not fail the upsert and end the sync. - let display_name = match &rt.display_name { - Some(Some(name)) if name.chars().count() > 100 => { - tracing::warn!( - "Ignoring the display_name of resource type {}: longer than 100 characters", - rt.name - ); - stored_display_name.clone() - } - Some(from_cache) => from_cache.clone(), - None => stored_display_name.clone(), - }; - - if let Some((existing_schema, existing_desc, _, _, _)) = existing { - if existing_schema == &rt.schema - && existing_desc == &rt.description - && stored_extension == format_extension - && stored_display_name == display_name - { - skipped_count += 1; - continue; - } - } - - // Insert or update resource type - sqlx::query( - // `format_extension` is resolved above rather than coalesced here: a - // COALESCE could never clear one, so a hub that dropped an extension - // would leave the stale value behind forever. - "INSERT INTO resource_type (workspace_id, name, schema, description, format_extension, display_name, edited_at) - VALUES ('admins', $1, $2, $3, $4, $5, now()) - ON CONFLICT (workspace_id, name) DO UPDATE - SET schema = EXCLUDED.schema, description = EXCLUDED.description, - format_extension = EXCLUDED.format_extension, - display_name = EXCLUDED.display_name, edited_at = now()", - ) - .bind(&rt.name) - .bind(&rt.schema) - .bind(&rt.description) - .bind(&format_extension) - .bind(&display_name) - .execute(db) - .await - .with_context(|| format!("Failed to upsert resource type {}", rt.name))?; - - synced_count += 1; - } - - tracing::info!( - "Synced {} resource types to admins workspace ({} skipped as unchanged)", - synced_count, - skipped_count - ); - - Ok(()) -} - fn print_help() { println!("Windmill - a fast, open-source workflow engine and job runner."); println!(); @@ -673,6 +437,8 @@ fn print_help() { println!(" cache-rt Pre-cache hub resource types"); println!(" sync-config Sync instance config from a YAML file to the database"); println!(" operator Run the Kubernetes operator (watches a ConfigMap)"); + #[cfg(all(feature = "enterprise", feature = "openidconnect", feature = "private"))] + println!(" oidc-signer (ee) Check an external OIDC signer, or run a file-backed one for tests"); println!(); println!("Environment variables (name = default):"); println!(" DATABASE_URL = The Postgres database url."); @@ -699,7 +465,7 @@ fn print_help() { println!(" LICENSE_KEY = None (EE only) Enterprise license key (workers require valid key)"); println!(" RUN_UPDATE_CA_CERTIFICATE_AT_START = false Run system CA update at startup"); println!(" RUN_UPDATE_CA_CERTIFICATE_PATH = /usr/sbin/update-ca-certificates Path to CA update tool"); - println!(" SYNC_CACHED_RT = false Sync cached resource types to admins workspace on server start"); + println!(" SYNC_CACHED_RT = false Sync the image's cached resource types to the admins workspace on server start"); println!(" HUB_BASE_URL = https://hub.windmill.dev Hub to fetch scripts from in `cache` mode (server/worker use the DB setting instead)"); println!(); println!("Notes:"); @@ -707,6 +473,38 @@ fn print_help() { println!("- At startup, Windmill logs currently set configuration keys for visibility."); } +async fn otel_environment(conn: &Connection) -> String { + if let Ok(environment) = std::env::var("OTEL_ENVIRONMENT") { + environment + } else { + load_base_url(conn) + .await + .unwrap_or_else(|_| "local".to_string()) + .trim_start_matches("https://") + .trim_start_matches("http://") + .split(".") + .next() + .unwrap_or_else(|| "local") + .to_string() + } +} + +/// Tracing for a subcommand that writes instance config. It never runs the server startup +/// that loads the OTel settings, and its audit events are exported under them. +/// +/// The subscriber it installs depends on those settings, so `early_logs` is what prints +/// the events of connecting and loading them; it gives way once the real one is in. +async fn init_declarative_writer_tracing( + db: &windmill_common::DB, + service: &str, + early_logs: tracing::subscriber::DefaultGuard, +) { + load_otel(db).await; + let environment = otel_environment(&Connection::Sql(db.clone())).await; + drop(early_logs); + windmill_common::tracing_init::initialize_subcommand_tracing(service, &HOSTNAME, &environment); +} + async fn windmill_main() -> anyhow::Result<()> { let (killpill_tx, mut killpill_rx) = KillpillSender::new(2); let mut monitor_killpill_rx = killpill_tx.subscribe(); @@ -810,7 +608,6 @@ async fn windmill_main() -> anyhow::Result<()> { return Ok(()); } "sync-config" => { - tracing_subscriber::fmt::init(); let path = std::env::args().nth(2).unwrap_or_else(|| { eprintln!("Usage: windmill sync-config "); std::process::exit(1); @@ -824,15 +621,40 @@ async fn windmill_main() -> anyhow::Result<()> { windmill_common::instance_config::resolve_env_refs(&mut config.global_settings) .map_err(|var| anyhow::anyhow!("environment variable '{var}' not found"))?; + let early_logs = tracing::subscriber::set_default(tracing_subscriber::fmt().finish()); tracing::info!("Connecting to database..."); let db = crate::db_connect::initial_connection().await?; - config.sync_to_db(&db).await?; + init_declarative_writer_tracing(&db, "windmill-sync-config", early_logs).await; + + let author = windmill_audit::instance_config::declarative_sync_author(); + let synced = config + .sync_to_db( + &db, + &windmill_audit::instance_config::InstanceConfigAuditor { + db: &db, + author: &author, + source: Some( + windmill_audit::instance_config::DeclarativeSource::SyncConfig, + ), + }, + ) + .await; + // Also on failure: the settings written before it are already audited, and the + // process exits before the exporter's next scheduled batch. + windmill_common::otel_oss::flush_otel_logs().await; + synced?; tracing::info!("Synced instance config from {path}"); return Ok(()); } + #[cfg(all(feature = "enterprise", feature = "openidconnect", feature = "private"))] + "oidc-signer" => { + tracing_subscriber::fmt::init(); + windmill_api::oidc_signer_tools_ee::run_cli(std::env::args().skip(2).collect()).await?; + return Ok(()); + } #[cfg(feature = "operator")] "operator" => { - tracing_subscriber::fmt::init(); + let early_logs = tracing::subscriber::set_default(tracing_subscriber::fmt().finish()); tracing::info!("Starting Windmill Kubernetes operator..."); tracing::info!("Connecting to database..."); @@ -845,6 +667,7 @@ async fn windmill_main() -> anyhow::Result<()> { operator_killpill_rx, ) .await?; + init_declarative_writer_tracing(&db, "windmill-operator", early_logs).await; #[cfg(all(feature = "enterprise", feature = "private"))] tokio::spawn(async move { @@ -971,19 +794,7 @@ async fn windmill_main() -> anyhow::Result<()> { (Connection::Sql(db), None, None) }; - let environment = if let Ok(environment) = std::env::var("OTEL_ENVIRONMENT") { - environment - } else { - load_base_url(&conn) - .await - .unwrap_or_else(|_| "local".to_string()) - .trim_start_matches("https://") - .trim_start_matches("http://") - .split(".") - .next() - .unwrap_or_else(|| "local") - .to_string() - }; + let environment = otel_environment(&conn).await; let _guard = windmill_common::tracing_init::initialize_tracing(&hostname, &mode, &environment); @@ -1018,8 +829,10 @@ async fn windmill_main() -> anyhow::Result<()> { .map(|v| v.to_lowercase() == "true" || v == "1") .unwrap_or(false) { - if let Err(e) = sync_cached_resource_types(db).await { - tracing::warn!("Failed to sync cached resource types: {:#}", e); + match hub_resource_types::sync_from_image_cache(db).await { + Ok(Some(outcome)) => tracing::info!("{}", outcome.summary()), + Ok(None) => {} + Err(e) => tracing::warn!("Failed to sync cached resource types: {:#}", e), } } } @@ -1237,6 +1050,15 @@ Windmill Community Edition {GIT_VERSION} // empty script cache, refilled from the database. windmill_common::cache::script::clear(); + // Before the first load: it can sign a token for instance object storage. + #[cfg(all(feature = "enterprise", feature = "openidconnect", feature = "private"))] + windmill_common::oidc_ee::init_oidc_signing( + &conn, + server_mode, + killpill_rx.resubscribe(), + ) + .await?; + initial_load( &conn, killpill_tx.clone(), @@ -1751,6 +1573,7 @@ Windmill Community Edition {GIT_VERSION} if server_mode { if let Some(db) = conn.as_sql() { schedule_stats(&db, &HTTP_CLIENT).await; + hub_resource_types::spawn_daily_sync(db.clone(), killpill_tx.subscribe()); } } @@ -1889,12 +1712,10 @@ async fn process_notify_event( ); windmill_common::workspaces::PUBLIC_APP_RATE_LIMIT_CACHE.remove(payload); } - windmill_common::SCRIPT_VERSION_DELETED_CHANNEL => { - match serde_json::from_str(payload) { - Ok(deleted) => windmill_api_scripts::scripts::evict_deleted_script_versions(deleted), - Err(e) => tracing::error!("Invalid script version deletion payload {payload}: {e}"), - } - } + windmill_common::SCRIPT_VERSION_DELETED_CHANNEL => match serde_json::from_str(payload) { + Ok(deleted) => windmill_api_scripts::scripts::evict_deleted_script_versions(deleted), + Err(e) => tracing::error!("Invalid script version deletion payload {payload}: {e}"), + }, "notify_runnable_version_change" => { tracing::info!("Runnable version change detected: {}", payload); match payload.split(':').collect::>().as_slice() { @@ -2055,6 +1876,11 @@ async fn process_notify_event( tracing::error!("Error loading preview tags override: {e:#}"); } } + DEPENDENCY_JOB_TAG_SETTING => { + if let Err(e) = load_dependency_job_tag(db).await { + tracing::error!("Error loading dependency job tag: {e:#}"); + } + } WORKSPACE_FAIRNESS_ENABLED_SETTING => { if let Err(e) = load_workspace_fairness_enabled(db).await { tracing::error!("Error loading workspace fairness enabled: {e:#}"); @@ -2322,6 +2148,12 @@ async fn process_notify_event( RESTART_COORDINATION_SETTING => { // Internal coordination key for staggered restarts, no action needed } + SYNC_HUB_RESOURCE_TYPES_DAILY_SETTING => { + // Read by the daily resource type sync each time it runs + } + "oidc_external_signer" => { + // Written by servers that sign with an external signer, read when signing + } "plain_emails_telemetry" => { let enabled = sqlx::query_scalar!( "SELECT value FROM global_settings WHERE name = 'plain_emails_telemetry'" @@ -2407,6 +2239,8 @@ pub async fn run_workers( &*PY311_CACHE_DIR, &*PY312_CACHE_DIR, &*PY313_CACHE_DIR, + &*PY314_CACHE_DIR, + &*PY315_CACHE_DIR, &*BUN_BUNDLE_CACHE_DIR, &*GO_CACHE_DIR, &*GO_BIN_CACHE_DIR, diff --git a/backend/src/monitor.rs b/backend/src/monitor.rs index 751d19fecc..23ab2eaed1 100644 --- a/backend/src/monitor.rs +++ b/backend/src/monitor.rs @@ -40,7 +40,7 @@ use windmill_common::ee_oss::{jobs_waiting_alerts, worker_groups_alerts}; use windmill_common::global_settings::OAUTH_SETTING; use windmill_common::otel_oss::{ otel_incr_zombie_delete_count, otel_incr_zombie_restart_count, otel_set_db_pool, - otel_set_queue_count, otel_set_queue_running_count, + otel_set_queue_count, otel_set_queue_delay, otel_set_queue_running_count, }; use windmill_common::{ agent_workers::DECODED_AGENT_TOKEN, @@ -60,17 +60,17 @@ use windmill_common::{ CRITICAL_ALERTS_ON_DB_OVERSIZE_SETTING, CRITICAL_ALERTS_ON_TOKEN_EXPIRY_SETTING, CRITICAL_ALERT_MUTE_UI_SETTING, CRITICAL_ALERT_MUTE_ZOMBIE_JOB_RESTART_SETTING, CRITICAL_ERROR_CHANNELS_SETTING, CUSTOM_TAGS_SETTING, DEFAULT_TAGS_PER_WORKSPACE_SETTING, - DEFAULT_TAGS_WORKSPACES_SETTING, DISABLE_PASSWORD_LOGIN, DISABLE_PASSWORD_LOGIN_SETTING, - EXPOSE_DEBUG_METRICS_SETTING, EXPOSE_METRICS_SETTING, EXTRA_PIP_INDEX_URL_SETTING, - FORK_WORKSPACE_TAG_APPEND_FORK_SUFFIX_SETTING, HUB_API_SECRET_SETTING, - HUB_BASE_URL_SETTING, INSTANCE_PYTHON_VERSION_SETTING, JOB_DEFAULT_TIMEOUT_SECS_SETTING, - JOB_ISOLATION_SETTING, JWT_SECRET_SETTING, KEEP_JOB_DIR_SETTING, LICENSE_KEY_SETTING, - MCP_DISABLE_TOKEN_QUERY_PARAM, MCP_DISABLE_TOKEN_QUERY_PARAM_SETTING, - MONITOR_LOGS_ON_OBJECT_STORE_SETTING, NPMRC_SETTING, NPM_CONFIG_REGISTRY_SETTING, - NSJAIL_TMPFS_SIZE_MB_SETTING, NSJAIL_TMP_BACKING_SETTING, NUGET_CONFIG_SETTING, - OTEL_SETTING, OTEL_TRACES_RETENTION_SECS_SETTING, OTEL_TRACING_PROXY_SETTING, - PIP_INDEX_URL_SETTING, POWERSHELL_REPO_PAT_SETTING, POWERSHELL_REPO_URL_SETTING, - PREVIEW_TAGS_OVERRIDE_SETTING, REQUEST_SIZE_LIMIT_SETTING, + DEFAULT_TAGS_WORKSPACES_SETTING, DEPENDENCY_JOB_TAG_SETTING, DISABLE_PASSWORD_LOGIN, + DISABLE_PASSWORD_LOGIN_SETTING, EXPOSE_DEBUG_METRICS_SETTING, EXPOSE_METRICS_SETTING, + EXTRA_PIP_INDEX_URL_SETTING, FORK_WORKSPACE_TAG_APPEND_FORK_SUFFIX_SETTING, + HUB_API_SECRET_SETTING, HUB_BASE_URL_SETTING, INSTANCE_PYTHON_VERSION_SETTING, + JOB_DEFAULT_TIMEOUT_SECS_SETTING, JOB_ISOLATION_SETTING, JWT_SECRET_SETTING, + KEEP_JOB_DIR_SETTING, LICENSE_KEY_SETTING, MCP_DISABLE_TOKEN_QUERY_PARAM, + MCP_DISABLE_TOKEN_QUERY_PARAM_SETTING, MONITOR_LOGS_ON_OBJECT_STORE_SETTING, NPMRC_SETTING, + NPM_CONFIG_REGISTRY_SETTING, NSJAIL_TMPFS_SIZE_MB_SETTING, NSJAIL_TMP_BACKING_SETTING, + NUGET_CONFIG_SETTING, OTEL_SETTING, OTEL_TRACES_RETENTION_SECS_SETTING, + OTEL_TRACING_PROXY_SETTING, PIP_INDEX_URL_SETTING, POWERSHELL_REPO_PAT_SETTING, + POWERSHELL_REPO_URL_SETTING, PREVIEW_TAGS_OVERRIDE_SETTING, REQUEST_SIZE_LIMIT_SETTING, REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING, RETENTION_PERIOD_SECS_SETTING, SAML_METADATA_SETTING, SANDBOX_IMAGE_CACHE_MAX_MB_SETTING, SANDBOX_IMAGE_DEFAULT_REGISTRY_SETTING, SANDBOX_IMAGE_MAX_SIZE_MB_SETTING, @@ -93,11 +93,11 @@ use windmill_common::{ load_periodic_bash_script_interval_from_env, load_whitelist_env_vars_from_env, load_worker_config, store_pull_query, store_suspended_pull_query, Connection, WorkerConfig, CLOUD_HOSTED, CONCURRENCY_KEY_MAX_QUEUED, CONCURRENCY_KEY_MAX_QUEUED_DEFAULT, - DEFAULT_TAGS_PER_WORKSPACE, DEFAULT_TAGS_WORKSPACES, FORK_WORKSPACE_TAG_APPEND_FORK_SUFFIX, - INDEXER_CONFIG, PREVIEW_TAGS_OVERRIDE, SMTP_CONFIG, WINDMILL_DIR, WORKER_CONFIG, - WORKER_GROUP, WORKSPACE_FAIRNESS_DURATION_SECS, WORKSPACE_FAIRNESS_ENABLED, - WORKSPACE_FAIRNESS_MAX_PERCENT, WORKSPACE_FAIRNESS_MIN_TOTAL, WORKSPACE_MAX_QUEUED_JOBS, - WORKSPACE_MAX_QUEUED_JOBS_DEFAULT, + DEFAULT_TAGS_PER_WORKSPACE, DEFAULT_TAGS_WORKSPACES, DEPENDENCY_JOB_TAG, + FORK_WORKSPACE_TAG_APPEND_FORK_SUFFIX, INDEXER_CONFIG, PREVIEW_TAGS_OVERRIDE, SMTP_CONFIG, + WINDMILL_DIR, WORKER_CONFIG, WORKER_GROUP, WORKSPACE_FAIRNESS_DURATION_SECS, + WORKSPACE_FAIRNESS_ENABLED, WORKSPACE_FAIRNESS_MAX_PERCENT, WORKSPACE_FAIRNESS_MIN_TOTAL, + WORKSPACE_MAX_QUEUED_JOBS, WORKSPACE_MAX_QUEUED_JOBS_DEFAULT, }, KillpillSender, AUDIT_LOG_RETENTION_DAYS, BASE_URL, CRITICAL_ALERTS_ON_DB_OVERSIZE, CRITICAL_ALERTS_ON_TOKEN_EXPIRY, CRITICAL_ALERT_MUTE_UI_ENABLED, @@ -168,6 +168,12 @@ lazy_static::lazy_static! { &["tag"] ).unwrap(); + static ref QUEUE_DELAY_SECONDS: prometheus::GaugeVec = prometheus::register_gauge_vec!( + "queue_delay_seconds", + "How long the next job to be picked up has been waiting, in seconds", + &["tag"] + ).unwrap(); + static ref QUEUE_RUNNING_COUNT: prometheus::IntGaugeVec = prometheus::register_int_gauge_vec!( "queue_running_count", "Number of running jobs in the queue", @@ -312,6 +318,9 @@ pub async fn initial_load( pass.setting(PREVIEW_TAGS_OVERRIDE_SETTING, false, |v| async move { apply_preview_tags_override(v) }); + pass.setting(DEPENDENCY_JOB_TAG_SETTING, false, |v| async move { + apply_dependency_job_tag(v) + }); // Load per-workspace retention overrides before the first cleanup tick so a fresh server // never sweeps globally without honoring configured longer-retention workspaces. @@ -418,6 +427,8 @@ pub async fn initial_load( native_mode, // an agent worker never reads its group's config, only its token object_store_cache_config: None, + // nor whether it is paused: the server refuses it jobs instead + paused: false, })); } } @@ -876,6 +887,24 @@ pub fn apply_preview_tags_override(value: Option) { } } +pub async fn load_dependency_job_tag(db: &DB) -> error::Result<()> { + let v = load_value_from_global_settings(db, DEPENDENCY_JOB_TAG_SETTING).await?; + apply_dependency_job_tag(v); + Ok(()) +} + +pub fn apply_dependency_job_tag(value: Option) { + let tag = match value { + Some(serde_json::Value::String(t)) if !t.trim().is_empty() => Some(t.trim().to_string()), + None | Some(serde_json::Value::Null) | Some(serde_json::Value::String(_)) => None, + Some(other) => { + tracing::error!("{DEPENDENCY_JOB_TAG_SETTING} is not a string: {other}, ignoring"); + return; + } + }; + DEPENDENCY_JOB_TAG.store(std::sync::Arc::new(tag)); +} + // Upper bound on the duration window. Postgres `make_interval(secs => $1::int4)` is the consumer // downstream, so this stays comfortably below `i32::MAX` and the subsequent `u32 -> i32` cast in // `workspace_fairness::refresh_overloaded` cannot wrap into a negative interval (which would @@ -4107,6 +4136,20 @@ pub async fn monitor_db( } _ => {} } + match windmill_common::runnable_job_stats::cleanup_old_runnable_job_stats( + db, + windmill_common::runnable_job_stats::RETENTION_DAYS, + ) + .await + { + Ok(count) if count > 0 => { + tracing::info!("Deleted {} old runnable job stats rows", count); + } + Err(e) => { + tracing::error!("Error cleaning up runnable job stats: {:?}", e); + } + _ => {} + } } } }; @@ -4406,6 +4449,42 @@ pub async fn monitor_db( } }; + // Drop the data table role set logins nobody has connected with for a month. Every ~40 min, + // spawned for the same reason as the credential maintenance above: it opens a connection per + // managed database for each login it drops. + let datatable_role_set_sweep_f = async { + if server_mode && iteration.is_some() && iteration.as_ref().unwrap().should_run(240) { + if let Some(db) = conn.as_sql() { + let db = db.clone(); + tokio::spawn(async move { + if let Err(e) = + windmill_common::datatable_roles::sweep_unused_role_set_logins(&db).await + { + tracing::error!("Error sweeping unused data table role set logins: {e}"); + } + }); + } + } + }; + + // Re-encrypt the legacy-format secret variables once new values are written in AES-256-GCM. + // Every ~5 min until one pass has gone through every workspace; spawned so a large instance + // does not hold up the join below. + let legacy_secret_upgrade_f = async { + if server_mode + && iteration.is_some() + && iteration.as_ref().unwrap().should_run(30) + && windmill_common::variables::legacy_secret_upgrade_pending() + { + if let Some(db) = conn.as_sql() { + let db = db.clone(); + tokio::spawn(async move { + windmill_common::variables::upgrade_legacy_secret_variables(&db).await + }); + } + } + }; + // Delete the AI session backups older than their workspace's retention. Every ~40 min // (240 iterations at the default 10 s, the most a u8 `should_run` counts): the retention // counts in days. Spawned for the same reason as the credential maintenance above, a @@ -4477,7 +4556,9 @@ pub async fn monitor_db( cleanup_scheduled_job_deletions_f, git_auto_pull_f, git_credential_maintenance_f, + datatable_role_set_sweep_f, ai_session_retention_f, + legacy_secret_upgrade_f, pipeline_freshness_watchdog_f, reconcile_unarmed_schedules_f, stranded_jobs_f, @@ -5576,6 +5657,7 @@ pub async fn expose_queue_metrics(db: &Pool) { for q in QUEUE_COUNT_TAGS.read().await.iter() { if queue_stats.get(q).is_none() { (*QUEUE_COUNT).with_label_values(&[q]).set(0); + (*QUEUE_DELAY_SECONDS).with_label_values(&[q]).set(0.0); } } } @@ -5584,6 +5666,7 @@ pub async fn expose_queue_metrics(db: &Pool) { for q in OTEL_QUEUE_COUNT_TAGS.read().await.iter() { if queue_stats.get(q).is_none() { otel_set_queue_count(q, 0); + otel_set_queue_delay(q, 0.0); } } } @@ -5599,6 +5682,9 @@ pub async fn expose_queue_metrics(db: &Pool) { if metrics_enabled { let metric = (*QUEUE_COUNT).with_label_values(&[tag]); metric.set(count as i64); + (*QUEUE_DELAY_SECONDS) + .with_label_values(&[tag]) + .set(stat.delay); tags_to_watch.push(tag.to_string()); } @@ -5606,6 +5692,7 @@ pub async fn expose_queue_metrics(db: &Pool) { otel_tags_to_watch.push(tag.to_string()); } otel_set_queue_count(tag, count as i64); + otel_set_queue_delay(tag, stat.delay); } if save_metrics { @@ -5656,8 +5743,13 @@ pub async fn reload_worker_config(db: &DB, tx: KillpillSender, kill_if_change: b .as_ref() .is_some_and(|dws| !dws.is_empty()); + // Pausing or resuming must not restart the workers, dedicated ones included: a reload + // whose only difference is the flag skips every restart below. + let only_pause_changed = wc.paused != config.paused + && WorkerConfig { paused: wc.paused, ..config.clone() } == **wc; + if **wc != config || has_dedicated { - if kill_if_change { + if kill_if_change && !only_pause_changed { if has_dedicated || wc.dedicated_worker != config.dedicated_worker || wc.dedicated_workers != config.dedicated_workers @@ -6146,6 +6238,21 @@ async fn handle_zombie_jobs(db: &Pool, base_internal_url: &str, node_n continue; } if let Some(job) = job.unwrap() { + // Read while the job is still queued: a re-run (perpetual, retry) takes its cap from + // here once the `job_perms` row is swept after completion. A failed read leaves the + // job for the next sweep rather than completing it with no cap. + let perms = + match windmill_common::auth::get_job_perms(db, &job.id, &job.workspace_id).await { + Ok(perms) => perms, + Err(e) => { + tracing::error!( + "Could not read the permissions of zombie job {}: {e:#}", + job.id + ); + continue; + } + }; + let job_token_scopes = perms.as_ref().and_then(|p| p.job_token_scopes.clone()); let label = ephemeral_script_token_label(&job.permissioned_as, &job.created_by); let token = create_token_for_owner( &db, @@ -6155,7 +6262,7 @@ async fn handle_zombie_jobs(db: &Pool, base_internal_url: &str, node_n job_token_expiry_secs(&db, &job.workspace_id).await, &job.permissioned_as_email, &job.id, - None, + perms, Some(format!("handle_zombie_jobs")), ) .await @@ -6177,10 +6284,12 @@ async fn handle_zombie_jobs(db: &Pool, base_internal_url: &str, node_n ); let memory_peak = job.memory_peak.unwrap_or(0); let (_, killpill_rx_never_used) = KillpillSender::new(1); + let mut completed = windmill_queue::MiniCompletedJob::from(job); + completed.job_token_scopes = job_token_scopes; let _ = handle_job_error( db, &client, - &windmill_queue::MiniCompletedJob::from(job), + &completed, memory_peak, None, error::Error::ExecutionErr(error_message.clone()), diff --git a/backend/summarized_schema.txt b/backend/summarized_schema.txt index 169eb81681..ac07dc06d6 100644 --- a/backend/summarized_schema.txt +++ b/backend/summarized_schema.txt @@ -97,7 +97,7 @@ draft: workspace_id(char), path(char), typ(draft_type), value(json), created_at( email_to_igroup: email(char), igroup(char) email_trigger: path(char), local_part(char), workspaced_local_part(bool), script_path(char), is_flow(bool), workspace_id(char), edited_by(char), email(char), edited_at(ts), extra_perms(jsonb), error_handler_path(char), error_handler_args(jsonb), retry(jsonb), mode(trigger_mode), labels(text[]) favorite: usr(char), workspace_id(char), path(char), favorite_kind(favorite_kind) -flow: workspace_id(char), path(char), summary(text), description(text), value(jsonb), edited_by(char), edited_at(ts), archived(bool), schema(json), extra_perms(jsonb), dependency_job(uuid), draft_only(bool), tag(char), ws_error_handler_muted(bool), dedicated_worker(bool), timeout(int), visible_to_runner_only(bool), concurrency_key(char), versions(bigint[]), on_behalf_of(varchar), on_behalf_of_email(text), lock_error_logs(text), labels(text[]) +flow: workspace_id(char), path(char), summary(text), description(text), value(jsonb), edited_by(char), edited_at(ts), archived(bool), schema(json), extra_perms(jsonb), dependency_job(uuid), draft_only(bool), tag(char), ws_error_handler_muted(bool), dedicated_worker(bool), timeout(int), visible_to_runner_only(bool), concurrency_key(char), versions(bigint[]), on_behalf_of(varchar), on_behalf_of_email(text), lock_error_logs(text), labels(text[]), job_token_scopes(text[]) FK: (workspace_id) -> workspace(id) flow_conversation: id(uuid), workspace_id(char), flow_path(char), title(char), created_at(ts), updated_at(ts), created_by(char), is_test(bool) FK: (workspace_id) -> workspace(id) @@ -132,7 +132,7 @@ input: id(uuid), workspace_id(char), runnable_id(char), runnable_type(runnable_t FK: (workspace_id) -> workspace(id) instance_group: name(char), summary(char), id(char), scim_display_name(char), external_id(char) job_logs: job_id(uuid), workspace_id(char), created_at(ts), logs(text), log_offset(int), log_file_index(text[]) -job_perms: job_id(uuid), email(char), username(char), is_admin(bool), is_operator(bool), created_at(ts), workspace_id(char), groups(text[]), folders(jsonb[]), end_user_email(char) +job_perms: job_id(uuid), email(char), username(char), is_admin(bool), is_operator(bool), created_at(ts), workspace_id(char), groups(text[]), folders(jsonb[]), end_user_email(char), job_token_scopes(text[]) job_resolution: job_id(uuid), workspace_id(char), resolved_at(ts), resolved_by(char), note(text), automatic(bool) job_result_stream: job_id(uuid), workspace_id(text), stream(text) job_result_stream_v2: job_id(uuid), workspace_id(text), stream(text), idx(int) @@ -180,10 +180,11 @@ resource_type: workspace_id(char), name(char), schema(jsonb), description(text), FK: (workspace_id) -> workspace(id) resume_job: id(uuid), job(uuid), flow(uuid), created_at(ts), value(jsonb), approver(char), resume_id(int), approved(bool) FK: (flow) -> v2_job_queue(id) +runnable_job_stats: hour(bigint), workspace_id(char), runnable_path(char), worker_group(text), job_count(int), total_duration_ms(bigint), max_memory_peak(int), sum_memory_peak(bigint), memory_sample_count(int), total_cpu_ms(bigint) runnable_settings: hash(bigint), debouncing_settings(bigint), concurrency_settings(bigint) schedule: workspace_id(char), path(char), edited_by(char), edited_at(ts), schedule(char), enabled(bool), script_path(char), args(jsonb), extra_perms(jsonb), is_flow(bool), email(char), error(text), timezone(char), on_failure(char), on_recovery(char), on_failure_times(int), on_failure_exact(bool), on_failure_extra_args(jsonb), on_recovery_times(int), on_recovery_extra_args(jsonb), ws_error_handler_muted(bool), retry(jsonb), summary(char), no_flow_overlap(bool), tag(char), paused_until(ts), on_success(char), on_success_extra_args(jsonb), cron_version(text), description(text), dynamic_skip(char), labels(text[]) FK: (workspace_id) -> workspace(id) -script: workspace_id(char), hash(bigint), path(char), parent_hashes(bigint[]), summary(text), description(text), content(text), created_by(char), created_at(ts), archived(bool), schema(json), deleted(bool), is_template(bool), extra_perms(jsonb), lock(text), lock_error_logs(text), language(script_lang), kind(script_kind), tag(char), draft_only(bool), envs(char), concurrent_limit(int), concurrency_time_window_s(int), cache_ttl(int), dedicated_worker(bool), ws_error_handler_muted(bool), priority(smallint), timeout(int), delete_after_use(bool), restart_unless_cancelled(bool), concurrency_key(char), visible_to_runner_only(bool), auto_kind(varchar), codebase(char), has_preprocessor(bool), schema_validation(bool), assets(jsonb), debounce_key(char), debounce_delay_s(int), cache_ignore_s3_path(bool), runnable_settings_handle(bigint), labels(text[]), on_behalf_of(varchar), on_behalf_of_email(text) +script: workspace_id(char), hash(bigint), path(char), parent_hashes(bigint[]), summary(text), description(text), content(text), created_by(char), created_at(ts), archived(bool), schema(json), deleted(bool), is_template(bool), extra_perms(jsonb), lock(text), lock_error_logs(text), language(script_lang), kind(script_kind), tag(char), draft_only(bool), envs(char), concurrent_limit(int), concurrency_time_window_s(int), cache_ttl(int), dedicated_worker(bool), ws_error_handler_muted(bool), priority(smallint), timeout(int), delete_after_use(bool), restart_unless_cancelled(bool), concurrency_key(char), visible_to_runner_only(bool), auto_kind(varchar), codebase(char), has_preprocessor(bool), schema_validation(bool), assets(jsonb), debounce_key(char), debounce_delay_s(int), cache_ignore_s3_path(bool), runnable_settings_handle(bigint), labels(text[]), on_behalf_of(varchar), on_behalf_of_email(text), job_token_scopes(text[]) FK: (workspace_id) -> workspace(id) skip_workspace_diff_tally: workspace_id(char), added_at(ts) sqs_trigger: path(char), queue_url(char), aws_resource_path(char), message_attributes(text[]), script_path(char), is_flow(bool), workspace_id(char), edited_by(char), email(char), edited_at(ts), extra_perms(jsonb), error(text), server_id(char), last_server_ping(ts), aws_auth_resource_type(aws_auth_resource_type), error_handler_path(char), error_handler_args(jsonb), retry(jsonb), mode(trigger_mode), labels(text[]) @@ -219,7 +220,7 @@ websocket_trigger: path(char), url(char), script_path(char), is_flow(bool), work windmill_migrations: name(text), created_at(ts) worker_group_job_stats: hour(bigint), worker_group(text), script_lang(char), workspace_id(char), job_count(int), total_duration_ms(bigint) FK: (workspace_id) -> workspace(id) -worker_ping: worker(char), worker_instance(char), ping_at(ts), started_at(ts), ip(char), jobs_executed(int), custom_tags(text[]), worker_group(char), dedicated_worker(char), wm_version(char), current_job_id(uuid), current_job_workspace_id(char), vcpus(bigint), memory(bigint), occupancy_rate(float), memory_usage(bigint), wm_memory_usage(bigint), occupancy_rate_15s(float), occupancy_rate_5m(float), occupancy_rate_30m(float), job_isolation(text), dedicated_workers(text[]) +worker_ping: worker(char), worker_instance(char), ping_at(ts), started_at(ts), ip(char), jobs_executed(int), custom_tags(text[]), worker_group(char), dedicated_worker(char), wm_version(char), current_job_id(uuid), current_job_workspace_id(char), vcpus(bigint), memory(bigint), occupancy_rate(float), memory_usage(bigint), wm_memory_usage(bigint), occupancy_rate_15s(float), occupancy_rate_5m(float), occupancy_rate_30m(float), job_isolation(text), dedicated_workers(text[]), draining(bool) workspace: id(char), name(char), owner(char), deleted(bool), premium(bool), parent_workspace_id(char), is_dev_workspace(bool) FK: (parent_workspace_id) -> workspace(id) workspace_dependencies: id(bigint), name(char), content(text), language(script_lang), description(text), archived(bool), workspace_id(char), created_at(ts) diff --git a/backend/tests/ai_agent_nul.rs b/backend/tests/ai_agent_nul.rs new file mode 100644 index 0000000000..1af46ecc82 --- /dev/null +++ b/backend/tests/ai_agent_nul.rs @@ -0,0 +1,125 @@ +use axum::{routing::post, Json, Router}; +use serde_json::{json, Value}; +use sqlx::{Pool, Postgres}; +use std::time::Duration; +use windmill_common::{flows::FlowValue, jobs::JobPayload}; +use windmill_test_utils::{ + completed_job, in_test_worker, listen_for_completed_jobs, ApiServer, RunJob, StreamFind, +}; + +const MEMORY_ID: &str = "0199c0de-0000-7000-8000-000000000001"; + +/// A model that puts a U+0000 in its tool call arguments, then in its answer. +async fn model(Json(body): Json) -> ([(&'static str, &'static str); 1], String) { + let finished = body["messages"] + .as_array() + .unwrap() + .iter() + .any(|m| m["role"] == "tool"); + let delta = if finished { + json!({"role":"assistant", "content":"are\u{0}tired"}) + } else { + json!({"role":"assistant", "tool_calls":[{ + "index":0, "id":"call_0", "type":"function", + "function":{"name":"echo", "arguments":"{\"x\":\"a\\u0000b\"}"} + }]}) + }; + let event = json!({"choices":[{"index":0, "delta":delta, "finish_reason":null}]}); + let end = json!({"choices":[{"index":0, "delta":{}, "finish_reason":if finished {"stop"} else {"tool_calls"}}]}); + ( + [("content-type", "text/event-stream")], + format!("data: {event}\n\ndata: {end}\n\ndata: [DONE]\n\n"), + ) +} + +async fn run_flow(db: &Pool, modules: Value) -> anyhow::Result { + let server = ApiServer::start(db.clone()).await?; + let flow: FlowValue = serde_json::from_value(json!({ "modules": modules }))?; + let id = RunJob::from(JobPayload::RawFlow { + value: flow, + path: Some("u/test/nul".into()), + restarted_from: None, + }) + .push(db) + .await; + let notifications = listen_for_completed_jobs(db).await; + tokio::time::timeout( + Duration::from_secs(45), + in_test_worker(db, notifications.find(&id), server.addr.port()), + ) + .await?; + let job = completed_job(id, db).await; + server.close().await?; + assert!(job.success, "{:?}", job.result); + Ok(job.json_result().expect("flow result")) +} + +#[sqlx::test(fixtures("base"))] +#[serial_test::serial] +async fn agent_step_survives_nul_in_model_output(db: Pool) -> anyhow::Result<()> { + std::env::set_var("ALLOW_PRIVATE_AI_BASE_URLS", "true"); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await?; + let base = format!("http://{}", listener.local_addr()?); + let stub = tokio::spawn(async move { + let router = Router::new().route("/v1/chat/completions", post(model)); + axum::serve(listener, router).await.unwrap(); + }); + let result = run_flow( + &db, + json!([{"id":"agent","value":{ + "type":"aiagent", + "tools":[{"id":"t0","summary":"echo","value":{ + "type":"rawscript", "language":"bun", "input_transforms":{}, + "content":"export async function main(x: string) { return x }" + }}], + "input_transforms":{ + "provider":{"type":"static","value":{"kind":"customai","model":"nul-test","resource":{"base_url":format!("{base}/v1")}}}, + "user_message":{"type":"static","value":"echo"}, + "memory":{"type":"static","value":{"kind":"auto","context_length":10,"memory_id":MEMORY_ID}} + } + }}]), + ) + .await?; + stub.abort(); + assert_eq!(result["output"], "aretired"); + let tool_message = result["messages"] + .as_array() + .unwrap() + .iter() + .find(|m| m["role"] == "tool") + .expect("tool message"); + assert_eq!(tool_message["content"], "\"ab\""); + // The model's own words are also written to a jsonb column the job result never passes through. + let memory: String = sqlx::query_scalar( + "SELECT messages::text FROM ai_agent_memory WHERE conversation_id = $1::uuid", + ) + .bind(MEMORY_ID) + .fetch_one(&db) + .await?; + assert!(memory.contains("aretired"), "{memory}"); + Ok(()) +} + +/// A step's args are evaluated from the previous step's result as the worker holds it, not as +/// the completed job stored it. +#[sqlx::test(fixtures("base"))] +#[serial_test::serial] +async fn flow_step_args_survive_nul_in_previous_result(db: Pool) -> anyhow::Result<()> { + let result = run_flow( + &db, + json!([ + {"id":"a","value":{ + "type":"rawscript", "language":"bun", "input_transforms":{}, + "content":"export async function main() { return 'a\\u0000b' }" + }}, + {"id":"b","value":{ + "type":"rawscript", "language":"bun", + "input_transforms":{"x":{"type":"javascript","expr":"results.a"}}, + "content":"export async function main(x: string) { return x }" + }} + ]), + ) + .await?; + assert_eq!(result, "ab"); + Ok(()) +} diff --git a/backend/tests/ai_decision.rs b/backend/tests/ai_decision.rs new file mode 100644 index 0000000000..d6359c1cea --- /dev/null +++ b/backend/tests/ai_decision.rs @@ -0,0 +1,148 @@ +//! AI decision steps run end to end against a stand-in for the decision endpoints: TypeSafe's +//! System One, Cloudflare Workers AI's, which serves each model at its own URL inside an API +//! envelope, and OpenAI's Decisions API. + +use axum::{routing::post, Json, Router}; +use serde_json::{json, Value}; +use sqlx::{Pool, Postgres}; +use windmill_common::{flows::FlowValue, jobs::JobPayload}; +use windmill_test_utils::*; + +/// Answers every question as a choice of `refund`, the way TypeSafe shapes an answer. +fn refund_answers(body: &Value) -> Value { + let answers = body["questions"] + .as_object() + .into_iter() + .flatten() + .map(|(name, _)| { + let answer = json!({ + "type": "choice", + "choice": "refund", + "probabilities": {"refund": 0.9, "bug": 0.1}, + "confidence": 0.8 + }); + (name.clone(), answer) + }) + .collect::>(); + json!({"model": body["model"], "answers": answers, "usage": {"input_tokens": 10}}) +} + +async fn start_decision_stub() -> anyhow::Result { + let app = Router::new() + .route( + "/v1/systemone", + post(|Json(body): Json| async move { Json(refund_answers(&body)) }), + ) + .route( + "/accounts/acc/ai/run/@cf/cloudflare/clef-flash", + post(|Json(body): Json| async move { + Json(json!({"result": refund_answers(&body), "success": true, "errors": []})) + }), + ) + // OpenAI's Decisions API takes the questions as a list and answers with one. + .route( + "/openai/v1/decisions", + post(|Json(body): Json| async move { + let answers = body["questions"] + .as_array() + .into_iter() + .flatten() + .map(|q| json!({"type": "choice", "name": q["name"], "choice": "refund"})) + .collect::>(); + Json(json!({"answers": answers})) + }), + ); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await?; + let port = listener.local_addr()?.port(); + tokio::spawn(async move { axum::serve(listener, app).await }); + Ok(port) +} + +/// A decision's answers reach later steps as `results..output`, which a Branch to one after it +/// routes on. +#[cfg(feature = "quickjs")] +#[sqlx::test(fixtures("base"))] +async fn test_ai_decision_answers_route_a_following_branch( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + // The stub listens on loopback, which the SSRF check refuses without this. + std::env::set_var("ALLOW_PRIVATE_AI_BASE_URLS", "true"); + let stub_port = start_decision_stub().await?; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + for provider in [ + json!({ + "kind": "typesafe", + "resource": {"api_key": "key", "base_url": format!("http://127.0.0.1:{stub_port}/v1")}, + "model": "jev-latest" + }), + json!({ + "kind": "cloudflare", + "resource": { + "token": "key", + "base_url": format!("http://127.0.0.1:{stub_port}/accounts/acc/ai/run/@cf/cloudflare") + }, + "model": "clef-flash" + }), + json!({ + "kind": "openai", + "resource": {"api_key": "key", "base_url": format!("http://127.0.0.1:{stub_port}/openai/v1")}, + "model": "gpt-6-luna" + }), + ] { + let flow: FlowValue = serde_json::from_value(json!({ + "modules": [ + {"id": "d", "value": { + "type": "aidecision", + "input_transforms": { + "provider": {"type": "static", "value": provider}, + "state": {"type": "static", "value": "I was charged twice"}, + "questions": {"type": "static", "value": {"intent": { + "type": "choice", + "instructions": "What does the customer want?", + "criteria": {"refund": "Money back", "bug": "Something is broken"} + }}} + } + }}, + {"id": "route", "value": { + "type": "branchone", + "branches": [ + { + "expr": "results.d.output.intent.choice === 'bug'", + "modules": [{"id": "b", "value": {"type": "identity"}}] + }, + { + "expr": "results.d.output.intent.choice === 'refund'", + "modules": [{"id": "r", "value": { + "type": "rawscript", + "language": "bun", + "content": "export function main(choice){ return 'handled ' + choice }", + "input_transforms": {"choice": { + "type": "javascript", + "expr": "results.d.output.intent.choice" + }} + }}] + } + ], + "default": [] + }} + ] + }))?; + + let job = run_job_in_new_worker_until_complete( + &db, + false, + JobPayload::RawFlow { value: flow, path: None, restarted_from: None }, + port, + ) + .await; + assert_eq!( + job.json_result(), + Some(json!("handled refund")), + "{provider}" + ); + } + Ok(()) +} diff --git a/backend/tests/app_ctx_inputs.rs b/backend/tests/app_ctx_inputs.rs new file mode 100644 index 0000000000..cf386adc89 --- /dev/null +++ b/backend/tests/app_ctx_inputs.rs @@ -0,0 +1,88 @@ +//! A context input a raw app runnable declares (`fields. = { type: "ctx", .. }`) is +//! filled by the server from the viewer's session. Whatever the request sends under that +//! name is discarded: a runnable filters rows on it, so a viewer who could set it would +//! read someone else's. +//! +//! Users from the `base` fixture: +//! test-user (admin, token SECRET_TOKEN) +//! test-user-2 (developer, token SECRET_TOKEN_2) + +use serde_json::json; +use sqlx::{Pool, Postgres}; +use windmill_test_utils::*; + +const ADMIN_TOKEN: &str = "SECRET_TOKEN"; +const VIEWER_TOKEN: &str = "SECRET_TOKEN_2"; +const APP_PATH: &str = "u/test-user/ctx_app"; +const CONTENT: &str = "export function main(who: string) { return who }"; +// sha256 of CONTENT, the key an inline runnable is pinned under. +const CONTENT_SHA: &str = "6e198199621d57e4fed834510cb0c2692274aa4cbe976e1704fab2806be9d744"; + +fn authed(builder: reqwest::RequestBuilder, token: &str) -> reqwest::RequestBuilder { + builder.header("Authorization", format!("Bearer {}", token)) +} + +#[sqlx::test(fixtures("base"))] +async fn a_declared_ctx_input_ignores_the_value_the_viewer_sends( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let ws = format!("http://localhost:{port}/api/w/test-workspace"); + let client = reqwest::Client::new(); + + let resp = authed(client.post(format!("{ws}/apps/create")), ADMIN_TOKEN) + .json(&json!({ + "path": APP_PATH, + "summary": "ctx inputs", + "value": { + "runnables": { + "whoami": { + "type": "inline", + "inlineScript": { "language": "bun", "content": CONTENT }, + "fields": { "who": { "type": "ctx", "ctx": "username" } } + } + } + }, + // The policy says nothing of context inputs: the server reads the + // declaration from the deployed runnable. + "policy": { + "execution_mode": "anonymous", + "triggerables_v2": { + format!("whoami:rawscript/{CONTENT_SHA}"): { + "static_inputs": {}, + "one_of_inputs": {} + } + } + } + })) + .send() + .await?; + assert!(resp.status().is_success(), "{}", resp.text().await?); + + // Both spellings a viewer could send: a forged literal, and no argument at all. + for args in [json!({ "who": "test-user" }), json!({})] { + let resp = authed( + client.post(format!("{ws}/apps_u/execute_component/{APP_PATH}")), + VIEWER_TOKEN, + ) + .json(&json!({ + "component": "whoami", + "args": args, + "raw_code": { "language": "bun", "content": CONTENT } + })) + .send() + .await?; + assert!(resp.status().is_success(), "{}", resp.text().await?); + let job_id: uuid::Uuid = resp.text().await?.parse()?; + + let queued: serde_json::Value = sqlx::query_scalar("SELECT args FROM v2_job WHERE id = $1") + .bind(job_id) + .fetch_one(&db) + .await?; + assert_eq!(queued["who"], json!("test-user-2"), "sent {args}"); + } + + Ok(()) +} diff --git a/backend/tests/asset_trigger_dispatch.rs b/backend/tests/asset_trigger_dispatch.rs index c165958a33..08d755f5d1 100644 --- a/backend/tests/asset_trigger_dispatch.rs +++ b/backend/tests/asset_trigger_dispatch.rs @@ -245,6 +245,7 @@ fn make_mini(id: Uuid, runnable_path: &str) -> MiniCompletedJob { cache_ignore_s3_path: None, runnable_settings_handle: None, build_binary_only: false, + job_token_scopes: None, } } @@ -330,6 +331,7 @@ async fn end_to_end_asset_dispatch(db: Pool) -> anyhow::Result<()> { concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default(), debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), labels: None, + job_token_scopes: None, }; let completed = RunJob::from(job).run_until_complete(&db, false, port).await; assert!( @@ -493,6 +495,7 @@ async fn partition_dynamic_resolved_persisted_and_propagated( concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default(), debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), labels: None, + job_token_scopes: None, }; let completed = RunJob::from(job) .arg("tenant_id", json!("acme")) diff --git a/backend/tests/batch_rerun.rs b/backend/tests/batch_rerun.rs index 51a6afa565..58cf27db87 100644 --- a/backend/tests/batch_rerun.rs +++ b/backend/tests/batch_rerun.rs @@ -53,6 +53,7 @@ fn script_payload() -> JobPayload { concurrency_settings: ConcurrencySettings::default().into(), debouncing_settings: DebouncingSettings::default(), labels: None, + job_token_scopes: None, } } @@ -63,6 +64,7 @@ fn flow_payload() -> JobPayload { apply_preprocessor: false, version: FLOW_VERSION, labels: None, + job_token_scopes: None, } } diff --git a/backend/tests/bun_jobs.rs b/backend/tests/bun_jobs.rs index 14f8c422a4..50fad28a04 100644 --- a/backend/tests/bun_jobs.rs +++ b/backend/tests/bun_jobs.rs @@ -854,6 +854,7 @@ fn run_main_script_job(hash: i64) -> RunJob { concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default(), debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), labels: None, + job_token_scopes: None, }) } diff --git a/backend/tests/caller_modules_strip.rs b/backend/tests/caller_modules_strip.rs index e6f96c04ec..a65ca22d1b 100644 --- a/backend/tests/caller_modules_strip.rs +++ b/backend/tests/caller_modules_strip.rs @@ -54,6 +54,7 @@ async fn stored_modules(db: &Pool, payload: JobPayload) -> Option) { concurrency_settings: ConcurrencySettings::default(), debouncing_settings: DebouncingSettings::default(), labels: None, + job_token_scopes: None, }; assert_eq!(stored_modules(&db, deployed).await, None); diff --git a/backend/tests/debounce_e2e.rs b/backend/tests/debounce_e2e.rs index 9a2351f135..709629d825 100644 --- a/backend/tests/debounce_e2e.rs +++ b/backend/tests/debounce_e2e.rs @@ -33,7 +33,7 @@ mod debounce_e2e { "id": "a", "value": { "type": "rawscript", - "language": "deno", + "language": "bun", "content": "export async function main(items: any[]) { return items }", "input_transforms": { "items": { "type": "javascript", "expr": "flow_input.items" } diff --git a/backend/tests/dedicated_workers.rs b/backend/tests/dedicated_workers.rs index 13fb043454..5e9c371f90 100644 --- a/backend/tests/dedicated_workers.rs +++ b/backend/tests/dedicated_workers.rs @@ -85,6 +85,7 @@ mod dedicated_worker_tests { apply_preprocessor: false, labels: None, version: 3000000000000001, + job_token_scopes: None, }) .arg("x", json!(5)) .push(&db) @@ -122,6 +123,7 @@ mod dedicated_worker_tests { apply_preprocessor: false, labels: None, version: 3000000000000002, + job_token_scopes: None, }) .arg("x", json!(7)) .push(&db) @@ -161,6 +163,7 @@ mod dedicated_worker_tests { apply_preprocessor: false, labels: None, version: 3000000000000003, + job_token_scopes: None, }) .arg("x", json!(5)) .push(&db) @@ -205,6 +208,7 @@ mod dedicated_worker_tests { concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default( ), debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), + job_token_scopes: None, }) .arg("x", json!(21)) .push(&db) @@ -247,6 +251,7 @@ mod dedicated_worker_tests { concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default( ), debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), + job_token_scopes: None, }) .arg("x", json!(5)) .push(&db) @@ -265,6 +270,7 @@ mod dedicated_worker_tests { concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default( ), debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), + job_token_scopes: None, }) .arg("x", json!(5)) .push(&db) @@ -327,6 +333,7 @@ mod dedicated_worker_tests { apply_preprocessor: false, labels: None, version: 3000000000000004, + job_token_scopes: None, }) .run_until_complete(&db, false, port) .await @@ -356,6 +363,7 @@ mod dedicated_worker_tests { apply_preprocessor: false, labels: None, version: 3000000000000005, + job_token_scopes: None, }) .arg("x", json!(5)) .push(&db) @@ -391,6 +399,7 @@ mod dedicated_worker_tests { apply_preprocessor: false, labels: None, version: 3000000000000006, + job_token_scopes: None, }) .arg("x", json!(5)) .push(&db) @@ -427,6 +436,7 @@ mod dedicated_worker_tests { apply_preprocessor: false, labels: None, version: 3000000000000007, + job_token_scopes: None, }) .arg("x", json!(5)) .push(&db) @@ -465,6 +475,7 @@ mod dedicated_worker_tests { apply_preprocessor: false, labels: None, version: 3000000000000008, + job_token_scopes: None, }) .arg("x", json!(5)) .push(&db) @@ -500,6 +511,7 @@ mod dedicated_worker_tests { apply_preprocessor: false, labels: None, version: 3000000000000009, + job_token_scopes: None, }) .arg("x", json!(5)) .push(&db) @@ -511,6 +523,7 @@ mod dedicated_worker_tests { apply_preprocessor: false, labels: None, version: 3000000000000010, + job_token_scopes: None, }) .arg("x", json!(5)) .push(&db) @@ -564,6 +577,7 @@ mod dedicated_worker_tests { concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default( ), debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), + job_token_scopes: None, }) .arg("x", json!(5)) .push(&db) @@ -622,6 +636,7 @@ mod dedicated_worker_tests { concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default( ), debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), + job_token_scopes: None, }) .arg("x", json!(7)) .push(&db) @@ -634,6 +649,7 @@ mod dedicated_worker_tests { apply_preprocessor: false, labels: None, version: 3000000000000011, + job_token_scopes: None, }) .arg("x", json!(7)) .push(&db) @@ -683,6 +699,7 @@ mod dedicated_worker_tests { apply_preprocessor: false, labels: None, version: 3000000000000012, + job_token_scopes: None, }) .arg("x", json!(5)) .push(&db) @@ -742,6 +759,7 @@ mod dedicated_worker_tests { apply_preprocessor: false, labels: None, version: 3000000000000013, + job_token_scopes: None, }) .run_until_complete(&db, false, port) .await @@ -777,6 +795,7 @@ mod dedicated_worker_tests { concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default( ), debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), + job_token_scopes: None, }) .arg("x", json!(5)) .push(&db) @@ -795,6 +814,7 @@ mod dedicated_worker_tests { concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default( ), debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), + job_token_scopes: None, }) .arg("x", json!(5)) .push(&db) @@ -848,6 +868,7 @@ mod dedicated_worker_tests { concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default( ), debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), + job_token_scopes: None, }) .arg("x", json!(5)) .push(&db) @@ -867,6 +888,7 @@ mod dedicated_worker_tests { concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default( ), debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), + job_token_scopes: None, }) .arg("x", json!(5)) .push(&db) @@ -926,6 +948,7 @@ mod dedicated_worker_tests { apply_preprocessor: false, labels: None, version: 3000000000000014, + job_token_scopes: None, }) .arg("x", json!(20)) .push(&db) @@ -938,6 +961,7 @@ mod dedicated_worker_tests { apply_preprocessor: false, labels: None, version: 3000000000000014, + job_token_scopes: None, }) .arg("x", json!(5)) .push(&db) @@ -988,6 +1012,7 @@ mod dedicated_worker_tests { concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default( ), debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), + job_token_scopes: None, }) .arg("x", json!(5)) .push(&db) @@ -1038,6 +1063,7 @@ mod dedicated_worker_tests { concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default( ), debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), + job_token_scopes: None, }) .arg("x", json!(5)) .push(&db) @@ -1089,6 +1115,7 @@ mod dedicated_worker_tests { concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default( ), debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), + job_token_scopes: None, }) .arg("x", json!(5)) .push(&db) @@ -1142,6 +1169,7 @@ mod dedicated_worker_tests { concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default( ), debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), + job_token_scopes: None, }) .arg("x", json!(5)) .push(&db) @@ -1160,6 +1188,7 @@ mod dedicated_worker_tests { concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default( ), debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), + job_token_scopes: None, }) .arg("x", json!(5)) .push(&db) @@ -1222,6 +1251,7 @@ mod dedicated_worker_tests { concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default( ), debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), + job_token_scopes: None, }) .arg("x", json!(7)) .push(&db) @@ -1240,6 +1270,7 @@ mod dedicated_worker_tests { concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default( ), debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), + job_token_scopes: None, }) .workspace("test-workspace-2") .arg("x", json!(7)) @@ -1294,6 +1325,7 @@ mod dedicated_worker_tests { concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default( ), debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), + job_token_scopes: None, }) .arg("x", json!(5)) .push(&db) @@ -1338,6 +1370,7 @@ mod dedicated_worker_tests { concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default( ), debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), + job_token_scopes: None, }) .arg("x", json!(5)) .push(&db) @@ -1375,6 +1408,7 @@ mod dedicated_worker_tests { apply_preprocessor: false, labels: None, version: 3000000000000015, + job_token_scopes: None, }) .push(&db) .await; @@ -1412,6 +1446,7 @@ mod dedicated_worker_tests { apply_preprocessor: false, labels: None, version: 3000000000000019, + job_token_scopes: None, }) .push(&db) .await; @@ -1451,6 +1486,7 @@ mod dedicated_worker_tests { apply_preprocessor: false, labels: None, version: 3000000000000016, + job_token_scopes: None, }) .arg("x", json!(1)) .push(&db) @@ -1490,6 +1526,7 @@ mod dedicated_worker_tests { apply_preprocessor: false, labels: None, version: 3000000000000017, + job_token_scopes: None, }) .arg("x", json!(5)) .push(&db) @@ -1529,6 +1566,7 @@ mod dedicated_worker_tests { apply_preprocessor: false, labels: None, version: 3000000000000018, + job_token_scopes: None, }) .push(&db) .await; diff --git a/backend/tests/dependency_job_tag.rs b/backend/tests/dependency_job_tag.rs new file mode 100644 index 0000000000..58b320d2c1 --- /dev/null +++ b/backend/tests/dependency_job_tag.rs @@ -0,0 +1,126 @@ +//! Sets the process-wide `DEPENDENCY_JOB_TAG`, so it lives in its own test binary: any other +//! test queuing a dependency job in the same process would land on the overridden tag. + +use sqlx::{Pool, Postgres}; +use windmill_common::worker::DEPENDENCY_JOB_TAG; +use windmill_dep_map::scoped_dependency_map::DependencyDependent; + +#[sqlx::test(fixtures("base"))] +async fn relocks_go_to_the_configured_dependency_job_tag(db: Pool) -> anyhow::Result<()> { + DEPENDENCY_JOB_TAG.store(std::sync::Arc::new(Some("deps-$workspace".to_string()))); + + for (path, hash, tag) in [ + ("f/tags/tagged", 7_100_001_i64, Some("default-$workspace")), + ("f/tags/plain", 7_100_002_i64, None), + ] { + sqlx::query( + "INSERT INTO script(workspace_id, created_by, content, schema, summary, description, path, hash, language, tag) + VALUES ('test-workspace', 'test-user', 'def main(): pass', '{}', '', '', $1, $2, 'python3', $3)", + ) + .bind(path) + .bind(hash) + .bind(tag) + .execute(&db) + .await?; + } + let importers = ["f/tags/tagged", "f/tags/plain"] + .map(|p| DependencyDependent { + importer_path: p.to_string(), + importer_kind: "script".to_string(), + importer_node_ids: None, + }) + .to_vec(); + windmill_dep_map::trigger_dependents::trigger_dependents_to_recompute_dependencies( + "test-workspace", + importers, + None, + None, + "test@windmill.dev", + "test-user", + "u/test-user", + &db, + vec![], + ) + .await?; + + let tags: Vec<(String, String)> = sqlx::query_as( + "SELECT runnable_path, tag FROM v2_job WHERE kind = 'dependencies' ORDER BY runnable_path", + ) + .fetch_all(&db) + .await?; + assert_eq!( + tags, + vec![ + ( + "f/tags/plain".to_string(), + "deps-test-workspace".to_string() + ), + ( + "f/tags/tagged".to_string(), + "deps-test-workspace".to_string() + ), + ] + ); + + // A binary prebuild given `auto_build_binary_tag` keeps it; an untagged one follows its + // dependency job. + assert_eq!( + push_binary_prebuild(&db, Some("build-pool")).await?, + "build-pool" + ); + assert_eq!( + push_binary_prebuild(&db, None).await?, + "deps-test-workspace" + ); + Ok(()) +} + +async fn push_binary_prebuild(db: &Pool, tag: Option<&str>) -> anyhow::Result { + let mut args = std::collections::HashMap::new(); + args.insert( + "build_binary_only".to_string(), + windmill_common::worker::to_raw_value(&true), + ); + let (build_id, tx) = windmill_queue::push( + db, + windmill_queue::PushIsolationLevel::IsolatedRoot(db.clone()), + "test-workspace", + windmill_common::jobs::JobPayload::BuildBinary { + path: "f/tags/plain".to_string(), + hash: windmill_common::scripts::ScriptHash(7_100_002), + language: windmill_common::scripts::ScriptLang::Go, + }, + windmill_queue::PushArgs::from(&args), + "test-user", + "test@windmill.dev", + "u/test-user".to_string(), + None, + None, + None, + None, + None, + None, + None, + None, + false, + false, + None, + true, + tag.map(str::to_string), + None, + None, + None, + None, + false, + None, + None, + None, + None, + ) + .await?; + tx.commit().await?; + Ok(sqlx::query_scalar("SELECT tag FROM v2_job WHERE id = $1") + .bind(build_id) + .fetch_one(db) + .await?) +} diff --git a/backend/tests/dependency_map.rs b/backend/tests/dependency_map.rs index 59895b7a3a..dad6c86aa3 100644 --- a/backend/tests/dependency_map.rs +++ b/backend/tests/dependency_map.rs @@ -452,6 +452,7 @@ def main(): ws_error_handler_muted: None, labels: None, skip_draft_deletion: None, + job_token_scopes: None, }) .send() .await diff --git a/backend/tests/drafts_delete_for_user.rs b/backend/tests/drafts_delete_for_user.rs new file mode 100644 index 0000000000..9c48f8828c --- /dev/null +++ b/backend/tests/drafts_delete_for_user.rs @@ -0,0 +1,85 @@ +//! Only a workspace admin can delete a draft that belongs to another user. +//! +//! A discard through `drafts/update` is scoped to the caller's own row, so it +//! never reaches someone else's draft. `drafts/delete_for_user` is the admin +//! route for that, and it must stay closed to everyone else. + +use serde_json::{json, Value}; +use sqlx::{Pool, Postgres}; + +use windmill_test_utils::*; + +#[sqlx::test(fixtures("base"))] +async fn test_only_admin_deletes_another_users_draft(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let base = format!("http://localhost:{port}/api/w/test-workspace/drafts"); + let client = reqwest::Client::new(); + let path = "script/u/test-user-2/theirs"; + + let save = |token: &'static str| { + client + .post(format!("{base}/update/{path}")) + .header("Authorization", format!("Bearer {token}")) + .json(&json!({ "value": { "path": "u/test-user-2/theirs", "content": "" } })) + .send() + }; + let resp = save("SECRET_TOKEN_2").await?; + assert_eq!(resp.status(), 200, "{}", resp.text().await?); + + let delete = |token: &'static str| { + client + .post(format!( + "{base}/delete_for_user/{path}?username=test-user-2" + )) + .header("Authorization", format!("Bearer {token}")) + .send() + }; + let own_draft = || async { + client + .get(format!("{base}/get_own/{path}")) + .header("Authorization", "Bearer SECRET_TOKEN_2") + .send() + .await? + .json::() + .await + }; + + let resp = delete("SECRET_TOKEN_3").await?; + assert_eq!(resp.status(), 403, "{}", resp.text().await?); + assert!( + !own_draft().await?.is_null(), + "a non-admin deleted the draft" + ); + + let resp = delete("SECRET_TOKEN").await?; + assert_eq!(resp.status(), 200, "{}", resp.text().await?); + assert!(own_draft().await?.is_null(), "the draft is still there"); + + let resp = delete("SECRET_TOKEN").await?; + assert_eq!(resp.status(), 404, "{}", resp.text().await?); + + // Without a username the delete takes every draft at the path. + for token in ["SECRET_TOKEN_2", "SECRET_TOKEN"] { + let resp = save(token).await?; + assert_eq!(resp.status(), 200, "{}", resp.text().await?); + } + let delete_all = |token: &'static str| { + client + .post(format!("{base}/delete_for_user/{path}")) + .header("Authorization", format!("Bearer {token}")) + .send() + }; + let resp = delete_all("SECRET_TOKEN_3").await?; + assert_eq!(resp.status(), 403, "{}", resp.text().await?); + let resp = delete_all("SECRET_TOKEN").await?; + assert_eq!(resp.status(), 200, "{}", resp.text().await?); + let left: i64 = sqlx::query_scalar( + "SELECT count(*) FROM draft WHERE workspace_id = 'test-workspace' AND path = 'u/test-user-2/theirs'", + ) + .fetch_one(&db) + .await?; + assert_eq!(left, 0, "drafts survived a path-wide delete"); + Ok(()) +} diff --git a/backend/tests/end_user_email.rs b/backend/tests/end_user_email.rs index 366c220c62..0d8ff7be9a 100644 --- a/backend/tests/end_user_email.rs +++ b/backend/tests/end_user_email.rs @@ -1,10 +1,8 @@ //! Tests for WM_END_USER_EMAIL environment variable. //! //! These tests verify that WM_END_USER_EMAIL is populated with the authenticated -//! user's email when executing app components. -//! -//! TODO: Add tests for scripts and flows once public execution endpoints are identified. -//! Currently only apps support non-workspace-member execution via OptAuthed + token lookup. +//! user's email when executing app components and when running scripts and flows directly. +//! Only apps support non-workspace-member execution via OptAuthed + token lookup. use serde_json::json; use sqlx::{Pool, Postgres}; @@ -27,40 +25,6 @@ fn authed(builder: reqwest::RequestBuilder, token: &str) -> reqwest::RequestBuil builder.header("Authorization", format!("Bearer {}", token)) } -// TODO: Script tests - need to identify public execution endpoints for non-workspace-members -// async fn run_script(port: u16, token: &str) -> anyhow::Result { -// let url = format!( -// "http://localhost:{}/api/w/test-workspace/jobs/run_wait_result/p/f/test/get_end_user_email", -// port -// ); -// let resp = authed(client().post(&url), token) -// .json(&json!({})) -// .send() -// .await?; -// if !resp.status().is_success() { -// anyhow::bail!("script run failed: {} - {}", resp.status(), resp.text().await?); -// } -// Ok(resp.json::().await? -// .as_str().unwrap_or("").to_string()) -// } - -// TODO: Flow tests - need to identify public execution endpoints for non-workspace-members -// async fn run_flow(port: u16, token: &str) -> anyhow::Result { -// let url = format!( -// "http://localhost:{}/api/w/test-workspace/jobs/run_wait_result/f/f/test/get_end_user_email_flow", -// port -// ); -// let resp = authed(client().post(&url), token) -// .json(&json!({})) -// .send() -// .await?; -// if !resp.status().is_success() { -// anyhow::bail!("flow run failed: {} - {}", resp.status(), resp.text().await?); -// } -// Ok(resp.json::().await? -// .as_str().unwrap_or("").to_string()) -// } - /// Create an app with inline script via API async fn create_app_with_inline_script(port: u16, path: &str) -> anyhow::Result<()> { let url = format!("http://localhost:{}/api/w/test-workspace/apps/create", port); @@ -74,8 +38,8 @@ async fn create_app_with_inline_script(port: u16, path: &str) -> anyhow::Result< "subgrids": {}, "hiddenInlineScripts": [{ "name": "get_email", - "language": "deno", - "content": "export function main() { return Deno.env.get(\"WM_END_USER_EMAIL\") || \"\"; }", + "language": "bun", + "content": "export function main() { return process.env.WM_END_USER_EMAIL || \"\"; }", "path": "f/test/email_app/get_email" }] }, @@ -89,7 +53,7 @@ async fn create_app_with_inline_script(port: u16, path: &str) -> anyhow::Result< "one_of_inputs": {} }, // SHA256 hash of raw_code content for anonymous execution - "rawscript/6428aba5aa2d3ea8e1215bfdccbedd3718b18da7a239e3778a9787bb9a0ea606": { + "rawscript/894d2f85ea2411e777e7d5787366c9fec4c1a7a5f83482748f1d129196c9f54f": { "static_inputs": {}, "one_of_inputs": {} } @@ -120,8 +84,8 @@ async fn create_raw_app_with_inline_script(port: u16, path: &str) -> anyhow::Res "css": "", "inlineScripts": [{ "name": "get_email", - "language": "deno", - "content": "export function main() { return Deno.env.get(\"WM_END_USER_EMAIL\") || \"\"; }" + "language": "bun", + "content": "export function main() { return process.env.WM_END_USER_EMAIL || \"\"; }" }] }, "policy": { @@ -134,7 +98,7 @@ async fn create_raw_app_with_inline_script(port: u16, path: &str) -> anyhow::Res "one_of_inputs": {} }, // SHA256 hash of raw_code content for anonymous execution - "rawscript/6428aba5aa2d3ea8e1215bfdccbedd3718b18da7a239e3778a9787bb9a0ea606": { + "rawscript/894d2f85ea2411e777e7d5787366c9fec4c1a7a5f83482748f1d129196c9f54f": { "static_inputs": {}, "one_of_inputs": {} } @@ -233,8 +197,8 @@ async fn run_app_inline_script( "args": {}, "component": "get_email", "raw_code": { - "language": "deno", - "content": "export function main() { return Deno.env.get(\"WM_END_USER_EMAIL\") || \"\"; }", + "language": "bun", + "content": "export function main() { return process.env.WM_END_USER_EMAIL || \"\"; }", "path": format!("{}/get_email", app_path) } }); @@ -270,8 +234,8 @@ async fn run_raw_app_inline_script( "args": {}, "component": "get_email", "raw_code": { - "language": "deno", - "content": "export function main() { return Deno.env.get(\"WM_END_USER_EMAIL\") || \"\"; }" + "language": "bun", + "content": "export function main() { return process.env.WM_END_USER_EMAIL || \"\"; }" } }); if force_viewer { @@ -312,41 +276,111 @@ async fn wait_for_job_result(port: u16, token: &str, job_id: &str) -> anyhow::Re anyhow::bail!("timeout waiting for job result") } -// TODO: Script tests - need to identify public execution endpoints for non-workspace-members -// #[cfg(feature = "deno_core")] -// #[sqlx::test(fixtures("base", "end_user_email"))] -// async fn test_script_wm_end_user_email(db: Pool) -> anyhow::Result<()> { -// initialize_tracing().await; -// set_jwt_secret().await; -// let server = ApiServer::start(db.clone()).await?; -// let port = server.addr.port(); -// -// in_test_worker(Connection::Sql(db.clone()), async move { -// let result = run_script(port, SAME_WS_TOKEN).await?; -// assert_eq!(result, SAME_WS_EMAIL, "same workspace user should get their email"); -// Ok::<(), anyhow::Error>(()) -// }, port).await?; -// -// Ok(()) -// } +async fn run_direct(port: u16, token: &str, route: &str) -> anyhow::Result { + let url = format!( + "http://localhost:{}/api/w/test-workspace/jobs/run/{}", + port, route + ); + let resp = authed(client().post(&url), token) + .json(&json!({})) + .send() + .await?; + if !resp.status().is_success() { + anyhow::bail!( + "run {route} failed: {} - {}", + resp.status(), + resp.text().await? + ); + } + let job_id = resp.text().await?; + wait_for_job_result(port, token, &job_id).await +} -// TODO: Flow tests - need to identify public execution endpoints for non-workspace-members -// #[cfg(feature = "deno_core")] -// #[sqlx::test(fixtures("base", "end_user_email"))] -// async fn test_flow_wm_end_user_email(db: Pool) -> anyhow::Result<()> { -// initialize_tracing().await; -// set_jwt_secret().await; -// let server = ApiServer::start(db.clone()).await?; -// let port = server.addr.port(); -// -// in_test_worker(Connection::Sql(db.clone()), async move { -// let result = run_flow(port, SAME_WS_TOKEN).await?; -// assert_eq!(result, SAME_WS_EMAIL, "same workspace user should get their email"); -// Ok::<(), anyhow::Error>(()) -// }, port).await?; -// -// Ok(()) -// } +#[cfg(feature = "deno_core")] +#[sqlx::test(fixtures("base", "end_user_email"))] +async fn test_direct_run_wm_end_user_email(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + set_jwt_secret().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + in_test_worker( + Connection::Sql(db.clone()), + async move { + let result = run_direct(port, SAME_WS_TOKEN, "p/f/test/get_end_user_email").await?; + assert_eq!(result, SAME_WS_EMAIL, "a script run should see its caller"); + + let result = + run_direct(port, SAME_WS_TOKEN, "f/f/test/get_end_user_email_flow").await?; + assert_eq!( + result, SAME_WS_EMAIL, + "every flow step should see the flow's caller" + ); + + Ok::<(), anyhow::Error>(()) + }, + port, + ) + .await?; + + Ok(()) +} + +/// A job's `WM_TOKEN` names the identity the job runs as, which differs from its end user when +/// the job runs on behalf of another account. A run it starts must keep the job's end user. +#[cfg(feature = "deno_core")] +#[sqlx::test(fixtures("base", "end_user_email"))] +async fn test_job_token_run_keeps_end_user_email(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + set_jwt_secret().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + let calling_job = uuid::Uuid::new_v4(); + sqlx::query( + "INSERT INTO job_perms (job_id, email, username, is_admin, is_operator, folders, groups, workspace_id, end_user_email) + VALUES ($1, 'test2@windmill.dev', 'test-user-2', false, false, '{}', '{}', 'test-workspace', $2)", + ) + .bind(calling_job) + .bind(SAME_WS_EMAIL) + .execute(&db) + .await?; + let wm_token = windmill_common::auth::create_jwt_token( + windmill_common::db::Authed { + email: "test2@windmill.dev".to_string(), + username: "test-user-2".to_string(), + is_admin: false, + is_operator: false, + groups: vec!["all".to_string()], + folders: vec![], + scopes: None, + token_prefix: None, + }, + "test-workspace", + 3600, + Some(calling_job), + None, + None, + None, + ) + .await?; + + in_test_worker( + Connection::Sql(db.clone()), + async move { + let result = run_direct(port, &wm_token, "p/f/test/get_end_user_email").await?; + assert_eq!( + result, SAME_WS_EMAIL, + "the calling job's end user, not its run-as identity" + ); + Ok::<(), anyhow::Error>(()) + }, + port, + ) + .await?; + + Ok(()) +} #[cfg(feature = "deno_core")] #[sqlx::test(fixtures("base", "end_user_email"))] diff --git a/backend/tests/error_handler.rs b/backend/tests/error_handler.rs index 5d5b7a828b..2e10006d2f 100644 --- a/backend/tests/error_handler.rs +++ b/backend/tests/error_handler.rs @@ -104,7 +104,7 @@ export async function main(path: string, email: string, job_id: string, is_flow: sqlx::query!( r#" INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by, schema, summary, description, lock) - VALUES ('test-workspace', 1111111111, 'f/test/error_handler', $1, 'deno', 'script', 'test-user', '{}', 'Error handler script', 'Handles failed job completions', '') + VALUES ('test-workspace', 1111111111, 'f/test/error_handler', $1, 'bun', 'script', 'test-user', '{}', 'Error handler script', 'Handles failed job completions', E'{}\n//bun.lock\n') "#, error_handler_code ) @@ -118,7 +118,7 @@ export async function main(path: string, email: string, job_id: string, is_flow: sqlx::query!( r#" INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by, schema, summary, description, lock) - VALUES ('test-workspace', $1, 'f/test/failing_script', $2, 'deno', 'script', 'test-user', '{}', 'Failing test script', 'A script that always fails', '') + VALUES ('test-workspace', $1, 'f/test/failing_script', $2, 'bun', 'script', 'test-user', '{}', 'Failing test script', 'A script that always fails', E'{}\n//bun.lock\n') "#, failing_script_hash, failing_script_code @@ -155,12 +155,13 @@ export async function main(path: string, email: string, job_id: string, is_flow: cache_ttl: None, cache_ignore_s3_path: None, dedicated_worker: None, - language: ScriptLang::Deno, + language: ScriptLang::Bun, priority: None, apply_preprocessor: false, concurrency_settings: ConcurrencySettings::default(), debouncing_settings: DebouncingSettings::default(), labels: None, + job_token_scopes: None, }) .run_until_complete(&db, false, server.addr.port()) .await; @@ -233,7 +234,7 @@ async fn test_error_handler_muted_on_script(db: Pool) -> anyhow::Resul sqlx::query!( r#" INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by, schema, summary, description, lock) - VALUES ('test-workspace', 3333333333, 'f/test/error_handler', 'export function main() { return "handled"; }', 'deno', 'script', 'test-user', '{}', '', '', '') + VALUES ('test-workspace', 3333333333, 'f/test/error_handler', 'export function main() { return "handled"; }', 'bun', 'script', 'test-user', '{}', '', '', E'{}\n//bun.lock\n') "#, ) .execute(&db) @@ -244,7 +245,7 @@ async fn test_error_handler_muted_on_script(db: Pool) -> anyhow::Resul sqlx::query!( r#" INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by, schema, summary, description, lock, ws_error_handler_muted) - VALUES ('test-workspace', $1, 'f/test/muted_failing_script', 'export function main() { throw new Error("fail"); }', 'deno', 'script', 'test-user', '{}', '', '', '', true) + VALUES ('test-workspace', $1, 'f/test/muted_failing_script', 'export function main() { throw new Error("fail"); }', 'bun', 'script', 'test-user', '{}', '', '', E'{}\n//bun.lock\n', true) "#, failing_script_hash, ) @@ -279,12 +280,13 @@ async fn test_error_handler_muted_on_script(db: Pool) -> anyhow::Resul cache_ttl: None, cache_ignore_s3_path: None, dedicated_worker: None, - language: ScriptLang::Deno, + language: ScriptLang::Bun, priority: None, apply_preprocessor: false, concurrency_settings: ConcurrencySettings::default(), debouncing_settings: DebouncingSettings::default(), labels: None, + job_token_scopes: None, }) .run_until_complete(&db, false, server.addr.port()) .await; @@ -329,7 +331,7 @@ async fn test_error_handler_not_triggered_on_success(db: Pool) -> anyh sqlx::query!( r#" INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by, schema, summary, description, lock) - VALUES ('test-workspace', 5555555555, 'f/test/error_handler', 'export function main() { return "handled"; }', 'deno', 'script', 'test-user', '{}', '', '', '') + VALUES ('test-workspace', 5555555555, 'f/test/error_handler', 'export function main() { return "handled"; }', 'bun', 'script', 'test-user', '{}', '', '', E'{}\n//bun.lock\n') "#, ) .execute(&db) @@ -340,7 +342,7 @@ async fn test_error_handler_not_triggered_on_success(db: Pool) -> anyh sqlx::query!( r#" INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by, schema, summary, description, lock) - VALUES ('test-workspace', $1, 'f/test/success_script', 'export function main() { return "ok"; }', 'deno', 'script', 'test-user', '{}', '', '', '') + VALUES ('test-workspace', $1, 'f/test/success_script', 'export function main() { return "ok"; }', 'bun', 'script', 'test-user', '{}', '', '', E'{}\n//bun.lock\n') "#, success_script_hash, ) @@ -375,12 +377,13 @@ async fn test_error_handler_not_triggered_on_success(db: Pool) -> anyh cache_ttl: None, cache_ignore_s3_path: None, dedicated_worker: None, - language: ScriptLang::Deno, + language: ScriptLang::Bun, priority: None, apply_preprocessor: false, concurrency_settings: ConcurrencySettings::default(), debouncing_settings: DebouncingSettings::default(), labels: None, + job_token_scopes: None, }) .run_until_complete(&db, false, server.addr.port()) .await; @@ -408,3 +411,70 @@ async fn test_error_handler_not_triggered_on_success(db: Pool) -> anyh Ok(()) } + +/// A hub script run at the top level (a schedule, a trigger, the API) is a `script_hub` job, +/// and its failure must reach the workspace error handler like a workspace script's. +#[cfg(all(feature = "enterprise", feature = "private"))] +#[sqlx::test(fixtures("base"))] +async fn test_error_handler_triggered_on_hub_script_failure( + db: Pool, +) -> anyhow::Result<()> { + use windmill_common::jobs::JobPayload; + + initialize_tracing().await; + + let server = ApiServer::start(db.clone()).await?; + + let version = "990000002"; + let hub_dir = &*windmill_common::worker::HUB_CACHE_DIR; + tokio::fs::create_dir_all(hub_dir).await?; + tokio::fs::write( + format!("{hub_dir}/{version}"), + r#"{"content":"exit 1","lockfile":null,"language":"bash","schema":{},"summary":null}"#, + ) + .await?; + + // The workspace handler settings are cached per workspace id across the tests of this + // binary, so this reuses the handler path every other test here configures. + sqlx::query( + "INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by, schema, summary, description, lock) + VALUES ('test-workspace', 1111111112, 'f/test/error_handler', 'echo handled', 'bash', 'script', 'test-user', '{}', '', '', '')", + ) + .execute(&db) + .await?; + sqlx::query( + r#"UPDATE workspace_settings SET error_handler = '{"path": "script/f/test/error_handler"}'::jsonb + WHERE workspace_id = 'test-workspace'"#, + ) + .execute(&db) + .await?; + + let completed_job = RunJob::from(JobPayload::ScriptHub { + path: format!("hub/{version}/test/fail"), + apply_preprocessor: false, + }) + .run_until_complete(&db, false, server.addr.port()) + .await; + assert!(!completed_job.success, "Hub script should have failed"); + + let mut handler_parent = None; + for _ in 0..50 { + handler_parent = sqlx::query_scalar::<_, Option>( + "SELECT parent_job FROM v2_job + WHERE workspace_id = 'test-workspace' AND permissioned_as_email = 'error_handler@windmill.dev'", + ) + .fetch_optional(&db) + .await?; + if handler_parent.is_some() { + break; + } + tokio::time::sleep(tokio::time::Duration::from_millis(100)).await; + } + assert_eq!( + handler_parent, + Some(Some(completed_job.id)), + "Workspace error handler should handle the failed hub script" + ); + + Ok(()) +} diff --git a/backend/tests/fixtures/end_user_email.sql b/backend/tests/fixtures/end_user_email.sql index 596d34e3e3..3b376525a4 100644 --- a/backend/tests/fixtures/end_user_email.sql +++ b/backend/tests/fixtures/end_user_email.sql @@ -40,9 +40,9 @@ VALUES (encode(sha256('NO_WS_TOKEN'::bytea), 'hex'), 'NO_WS_TOKE', 'NO_WS_TOKEN' INSERT INTO script (workspace_id, created_by, content, schema, summary, description, path, hash, language, lock, kind, extra_perms) VALUES ( 'test-workspace', 'test-user', - 'export function main() { return Deno.env.get("WM_END_USER_EMAIL") || ""; }', + 'export function main() { return process.env.WM_END_USER_EMAIL || ""; }', '{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{},"required":[],"type":"object"}', - 'Returns WM_END_USER_EMAIL', '', 'f/test/get_end_user_email', 900001, 'deno', '', 'script', + 'Returns WM_END_USER_EMAIL', '', 'f/test/get_end_user_email', 900001, 'bun', E'{}\n//bun.lock\n', 'script', '{"g/all": true}' ); @@ -51,7 +51,7 @@ INSERT INTO flow (workspace_id, summary, description, path, versions, schema, va VALUES ( 'test-workspace', 'Returns WM_END_USER_EMAIL', '', 'f/test/get_end_user_email_flow', '{900002}', '{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{},"required":[],"type":"object"}', - '{"modules": [{"id": "a", "value": {"type": "rawscript", "language": "deno", "content": "export function main() { return Deno.env.get(\"WM_END_USER_EMAIL\") || \"\"; }", "input_transforms": {}}}, {"id": "b", "value": {"type": "rawscript", "language": "deno", "content": "export function main() { return Deno.env.get(\"WM_END_USER_EMAIL\") || \"\"; }", "input_transforms": {}}}]}', + '{"modules": [{"id": "a", "value": {"type": "rawscript", "language": "bun", "content": "export function main() { return process.env.WM_END_USER_EMAIL || \"\"; }", "input_transforms": {}}}, {"id": "b", "value": {"type": "rawscript", "language": "bun", "content": "export function main() { return process.env.WM_END_USER_EMAIL || \"\"; }", "input_transforms": {}}}]}', 'test-user', '{"g/all": true}' ); @@ -60,6 +60,6 @@ INSERT INTO flow_version (id, workspace_id, path, schema, value, created_by) VALUES ( 900002, 'test-workspace', 'f/test/get_end_user_email_flow', '{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{},"required":[],"type":"object"}', - '{"modules": [{"id": "a", "value": {"type": "rawscript", "language": "deno", "content": "export function main() { return Deno.env.get(\"WM_END_USER_EMAIL\") || \"\"; }", "input_transforms": {}}}, {"id": "b", "value": {"type": "rawscript", "language": "deno", "content": "export function main() { return Deno.env.get(\"WM_END_USER_EMAIL\") || \"\"; }", "input_transforms": {}}}]}', + '{"modules": [{"id": "a", "value": {"type": "rawscript", "language": "bun", "content": "export function main() { return process.env.WM_END_USER_EMAIL || \"\"; }", "input_transforms": {}}}, {"id": "b", "value": {"type": "rawscript", "language": "bun", "content": "export function main() { return process.env.WM_END_USER_EMAIL || \"\"; }", "input_transforms": {}}}]}', 'test-user' ); diff --git a/backend/tests/fixtures/hello.sql b/backend/tests/fixtures/hello.sql index 962f6441d9..4c40ac3290 100644 --- a/backend/tests/fixtures/hello.sql +++ b/backend/tests/fixtures/hello.sql @@ -11,7 +11,7 @@ export function main(world: string) { '{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{"world":{"default":"world","description":"","type":"string"}},"required":[],"type":"object"}', '', '', -'f/system/hello', 123412, 'deno', ''); +'f/system/hello', 123412, 'bun', E'{}\n//bun.lock\n'); INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES ( 'test-workspace', @@ -28,7 +28,7 @@ export async function main(foo: string, bar: string) { '{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{"foo":{"default":null,"description":"","originalType":"string","type":"string"},"bar":{"default":null,"description":"","originalType":"string","type":"string"}},"required":["foo","bar"],"type":"object"}', '', '', -'f/system/hello_with_preprocessor', 123413, 'deno', ''); +'f/system/hello_with_preprocessor', 123413, 'bun', E'{}\n//bun.lock\n'); INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES ( 'test-workspace', @@ -140,7 +140,7 @@ INSERT INTO public.flow(workspace_id, summary, description, path, versions, sche 'f/system/hello_with_nodes_flow', '{1443253234253454}', '{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{"world":{"default":"world","description":"","type":"string"}},"required":[],"type":"object"}', -E'{"modules":[{"id":"a","value":{"type":"forloopflow","modules":[{"id":"b","value":{"type":"rawscript","content":"export function main(world: string) {\\n const greet = `Hello ${world}!`;\\n console.log(greet)\\n return greet\\n}\\n","language":"deno","input_transforms":{"world":{"type":"javascript","expr":"flow_input.iter.value"}},"is_trigger":false}},{"id":"c","value":{"type":"rawscript","content":"export function main(hello: string) {\\n const dareyou = `Did you just say \\"${hello}\\"??!`;\\n console.log(dareyou)\\n return dareyou\\n}","language":"deno","input_transforms":{"hello":{"type":"javascript","value":"${results.b}","expr":"`${results.b}`"}},"is_trigger":false}}],"iterator":{"type":"javascript","expr":"[''foo'', ''bar'', ''baz'']"},"skip_failures":true,"parallel":false}}],"same_worker":false}', +E'{"modules":[{"id":"a","value":{"type":"forloopflow","modules":[{"id":"b","value":{"type":"rawscript","content":"export function main(world: string) {\\n const greet = `Hello ${world}!`;\\n console.log(greet)\\n return greet\\n}\\n","language":"bun","input_transforms":{"world":{"type":"javascript","expr":"flow_input.iter.value"}},"is_trigger":false}},{"id":"c","value":{"type":"rawscript","content":"export function main(hello: string) {\\n const dareyou = `Did you just say \\"${hello}\\"??!`;\\n console.log(dareyou)\\n return dareyou\\n}","language":"bun","input_transforms":{"hello":{"type":"javascript","value":"${results.b}","expr":"`${results.b}`"}},"is_trigger":false}}],"iterator":{"type":"javascript","expr":"[''foo'', ''bar'', ''baz'']"},"skip_failures":true,"parallel":false}}],"same_worker":false}', 'system' ); @@ -149,7 +149,7 @@ INSERT INTO public.flow_version(id, workspace_id, path, schema, value, created_b 'test-workspace', 'f/system/hello_with_nodes_flow', '{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{"world":{"default":"world","description":"","type":"string"}},"required":[],"type":"object"}', -E'{"modules":[{"id":"a","value":{"type":"forloopflow","modules":[{"id":"b","value":{"type":"rawscript","content":"export function main(world: string) {\\n const greet = `Hello ${world}!`;\\n console.log(greet)\\n return greet\\n}\\n","language":"deno","input_transforms":{"world":{"type":"javascript","expr":"flow_input.iter.value"}},"is_trigger":false}},{"id":"c","value":{"type":"rawscript","content":"export function main(hello: string) {\\n const dareyou = `Did you just say \\"${hello}\\"??!`;\\n console.log(dareyou)\\n return dareyou\\n}","language":"deno","input_transforms":{"hello":{"type":"javascript","value":"${results.b}","expr":"`${results.b}`"}},"is_trigger":false}}],"iterator":{"type":"javascript","expr":"[''foo'', ''bar'', ''baz'']"},"skip_failures":true,"parallel":false}}],"same_worker":false}', +E'{"modules":[{"id":"a","value":{"type":"forloopflow","modules":[{"id":"b","value":{"type":"rawscript","content":"export function main(world: string) {\\n const greet = `Hello ${world}!`;\\n console.log(greet)\\n return greet\\n}\\n","language":"bun","input_transforms":{"world":{"type":"javascript","expr":"flow_input.iter.value"}},"is_trigger":false}},{"id":"c","value":{"type":"rawscript","content":"export function main(hello: string) {\\n const dareyou = `Did you just say \\"${hello}\\"??!`;\\n console.log(dareyou)\\n return dareyou\\n}","language":"bun","input_transforms":{"hello":{"type":"javascript","value":"${results.b}","expr":"`${results.b}`"}},"is_trigger":false}}],"iterator":{"type":"javascript","expr":"[''foo'', ''bar'', ''baz'']"},"skip_failures":true,"parallel":false}}],"same_worker":false}', 'system' ); @@ -160,7 +160,7 @@ INSERT INTO public.flow(workspace_id, summary, description, path, versions, sche 'f/system/hello_with_preprocessor', '{1443253234253456}', '{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{"foo":{"type":"string","description":"","default":""},"bar":{"type":"string","description":"","default":""}},"required":[],"type":"object","order":["foo","bar"]}', -E'{"modules":[{"id":"a","value":{"type":"rawscript","content":"export async function main(foo: string, bar: string) {\\n return ''Hello '' + foo + ''-'' + bar\\n}\\n","language":"deno","is_trigger":false,"input_transforms":{"bar":{"expr":"flow_input.bar","type":"javascript"},"foo":{"expr":"flow_input.foo","type":"javascript"}}}}],"preprocessor_module":{"id":"preprocessor","value":{"type":"rawscript","content":"export async function preprocessor(\\n\\twm_trigger: {\\n\\t\\tkind: ''http'' | ''email'' | ''webhook'' | ''websocket'' | ''kafka'' | ''nats'',\\n\\t\\thttp?: {\\n\\t\\t\\troute: string\\n\\t\\t\\tpath: string\\n\\t\\t\\tmethod: string\\n\\t\\t\\tparams: Record\\n\\t\\t\\tquery: Record\\n\\t\\t\\theaders: Record\\n\\t\\t},\\n\\t\\twebsocket?: {\\n\\t\\t\\turl: string\\n\\t\\t},\\n\\t\\tkafka?: {\\n\\t\\t\\tbrokers: string[]\\n\\t\\t\\ttopic: string\\n\\t\\t\\tgroup_id: string\\n\\t\\t},\\n\\t\\tnats?: {\\n\\t\\t\\tservers: string[]\\n\\t\\t\\tsubject: string\\n\\t\\t\\theaders?: Record\\n\\t\\t\\tstatus?: number\\n\\t\\t\\tdescription?: string\\n\\t\\t\\tlength: number\\n\\t\\t}\\n\\t},\\n) {\\n\\treturn {\\n\\t\\tfoo: ''bar'',\\n\\t\\tbar: ''baz''\\n\\t}\\n}","language":"deno","is_trigger":false,"input_transforms":{"wm_trigger":{"type":"static"}}}}}', +E'{"modules":[{"id":"a","value":{"type":"rawscript","content":"export async function main(foo: string, bar: string) {\\n return ''Hello '' + foo + ''-'' + bar\\n}\\n","language":"bun","is_trigger":false,"input_transforms":{"bar":{"expr":"flow_input.bar","type":"javascript"},"foo":{"expr":"flow_input.foo","type":"javascript"}}}}],"preprocessor_module":{"id":"preprocessor","value":{"type":"rawscript","content":"export async function preprocessor(\\n\\twm_trigger: {\\n\\t\\tkind: ''http'' | ''email'' | ''webhook'' | ''websocket'' | ''kafka'' | ''nats'',\\n\\t\\thttp?: {\\n\\t\\t\\troute: string\\n\\t\\t\\tpath: string\\n\\t\\t\\tmethod: string\\n\\t\\t\\tparams: Record\\n\\t\\t\\tquery: Record\\n\\t\\t\\theaders: Record\\n\\t\\t},\\n\\t\\twebsocket?: {\\n\\t\\t\\turl: string\\n\\t\\t},\\n\\t\\tkafka?: {\\n\\t\\t\\tbrokers: string[]\\n\\t\\t\\ttopic: string\\n\\t\\t\\tgroup_id: string\\n\\t\\t},\\n\\t\\tnats?: {\\n\\t\\t\\tservers: string[]\\n\\t\\t\\tsubject: string\\n\\t\\t\\theaders?: Record\\n\\t\\t\\tstatus?: number\\n\\t\\t\\tdescription?: string\\n\\t\\t\\tlength: number\\n\\t\\t}\\n\\t},\\n) {\\n\\treturn {\\n\\t\\tfoo: ''bar'',\\n\\t\\tbar: ''baz''\\n\\t}\\n}","language":"bun","is_trigger":false,"input_transforms":{"wm_trigger":{"type":"static"}}}}}', 'system' ); @@ -169,6 +169,6 @@ INSERT INTO public.flow_version(id, workspace_id, path, schema, value, created_b 'test-workspace', 'f/system/hello_with_preprocessor', '{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{"foo":{"type":"string","description":"","default":""},"bar":{"type":"string","description":"","default":""}},"required":[],"type":"object","order":["foo","bar"]}', -E'{"modules":[{"id":"a","value":{"type":"rawscript","content":"export async function main(foo: string, bar: string) {\\n return ''Hello '' + foo + ''-'' + bar\\n}\\n","language":"deno","is_trigger":false,"input_transforms":{"bar":{"expr":"flow_input.bar","type":"javascript"},"foo":{"expr":"flow_input.foo","type":"javascript"}}}}],"preprocessor_module":{"id":"preprocessor","value":{"type":"rawscript","content":"export async function preprocessor(\\n\\twm_trigger: {\\n\\t\\tkind: ''http'' | ''email'' | ''webhook'' | ''websocket'' | ''kafka'' | ''nats'',\\n\\t\\thttp?: {\\n\\t\\t\\troute: string\\n\\t\\t\\tpath: string\\n\\t\\t\\tmethod: string\\n\\t\\t\\tparams: Record\\n\\t\\t\\tquery: Record\\n\\t\\t\\theaders: Record\\n\\t\\t},\\n\\t\\twebsocket?: {\\n\\t\\t\\turl: string\\n\\t\\t},\\n\\t\\tkafka?: {\\n\\t\\t\\tbrokers: string[]\\n\\t\\t\\ttopic: string\\n\\t\\t\\tgroup_id: string\\n\\t\\t},\\n\\t\\tnats?: {\\n\\t\\t\\tservers: string[]\\n\\t\\t\\tsubject: string\\n\\t\\t\\theaders?: Record\\n\\t\\t\\tstatus?: number\\n\\t\\t\\tdescription?: string\\n\\t\\t\\tlength: number\\n\\t\\t}\\n\\t},\\n) {\\n\\treturn {\\n\\t\\tfoo: ''bar'',\\n\\t\\tbar: ''baz''\\n\\t}\\n}","language":"deno","is_trigger":false,"input_transforms":{"wm_trigger":{"type":"static"}}}}}', +E'{"modules":[{"id":"a","value":{"type":"rawscript","content":"export async function main(foo: string, bar: string) {\\n return ''Hello '' + foo + ''-'' + bar\\n}\\n","language":"bun","is_trigger":false,"input_transforms":{"bar":{"expr":"flow_input.bar","type":"javascript"},"foo":{"expr":"flow_input.foo","type":"javascript"}}}}],"preprocessor_module":{"id":"preprocessor","value":{"type":"rawscript","content":"export async function preprocessor(\\n\\twm_trigger: {\\n\\t\\tkind: ''http'' | ''email'' | ''webhook'' | ''websocket'' | ''kafka'' | ''nats'',\\n\\t\\thttp?: {\\n\\t\\t\\troute: string\\n\\t\\t\\tpath: string\\n\\t\\t\\tmethod: string\\n\\t\\t\\tparams: Record\\n\\t\\t\\tquery: Record\\n\\t\\t\\theaders: Record\\n\\t\\t},\\n\\t\\twebsocket?: {\\n\\t\\t\\turl: string\\n\\t\\t},\\n\\t\\tkafka?: {\\n\\t\\t\\tbrokers: string[]\\n\\t\\t\\ttopic: string\\n\\t\\t\\tgroup_id: string\\n\\t\\t},\\n\\t\\tnats?: {\\n\\t\\t\\tservers: string[]\\n\\t\\t\\tsubject: string\\n\\t\\t\\theaders?: Record\\n\\t\\t\\tstatus?: number\\n\\t\\t\\tdescription?: string\\n\\t\\t\\tlength: number\\n\\t\\t}\\n\\t},\\n) {\\n\\treturn {\\n\\t\\tfoo: ''bar'',\\n\\t\\tbar: ''baz''\\n\\t}\\n}","language":"bun","is_trigger":false,"input_transforms":{"wm_trigger":{"type":"static"}}}}}', 'system' ); diff --git a/backend/tests/fixtures/jobs_read_auth.sql b/backend/tests/fixtures/jobs_read_auth.sql index 980d68b711..e175a9c065 100644 --- a/backend/tests/fixtures/jobs_read_auth.sql +++ b/backend/tests/fixtures/jobs_read_auth.sql @@ -29,6 +29,13 @@ INSERT INTO token(token_hash, token_prefix, token, email, label, super_admin, sc ARRAY['jobs:run:flows:f/shared/flow1'] ); +-- A path-scoped read token for test-user-2: it reads the runs of `f/shared/flow1` only. +INSERT INTO token(token_hash, token_prefix, token, email, label, super_admin, scopes) VALUES ( + encode(sha256('READ_SCOPED_TOKEN'::bytea), 'hex'), 'READ_SCOP', 'READ_SCOPED_TOKEN', + 'test2@windmill.dev', 'flow history token', false, + ARRAY['jobs:read:f/shared/flow1'] +); + -- Same, scoped to a script. The two jobs below both run through a `singlestepflow` -- wrapper (native retry / scheduled runs produce these) — one wrapping a script, one -- wrapping a flow — so the confinement has to project each onto the runnable it wraps @@ -91,6 +98,32 @@ INSERT INTO public.v2_job_completed (id, workspace_id, duration_ms, status, resu ('18181818-1818-1818-1818-181818181818', 'test-workspace', 1000, 'success'::job_status, '{"agent": "AGENT_CHAT_RESULT"}'); +-- A lock job for flow `f/shared/flow1`, as `wmill generate-metadata` queues it, and the +-- token CI would lock with. +INSERT INTO token(token_hash, token_prefix, token, email, label, super_admin, scopes) VALUES ( + encode(sha256('RUN_SCOPED_DEPS_TOKEN'::bytea), 'hex'), 'RUN_DEPS_', 'RUN_SCOPED_DEPS_TOKEN', + 'test2@windmill.dev', 'lock token', false, + ARRAY['jobs:run:dependencies:f/shared/*'] +); +INSERT INTO public.v2_job ( + id, workspace_id, created_by, created_at, permissioned_as, permissioned_as_email, + kind, runnable_path, tag, visible_to_owner +) VALUES ( + '19191919-1919-1919-1919-191919191919', 'test-workspace', 'test-user-2', + '2023-01-01 00:00:00', 'u/test-user-2', 'test2@windmill.dev', + 'flowdependencies', 'f/shared/flow1', 'dependency', true +), ( + -- Queued by a broader token with `*` as its path, which the request body allows. + '20202020-2020-2020-2020-202020202020', 'test-workspace', 'test-user-2', + '2023-01-01 00:00:00', 'u/test-user-2', 'test2@windmill.dev', + 'dependencies', '*', 'dependency', true +); +INSERT INTO public.v2_job_completed (id, workspace_id, duration_ms, status, result) VALUES + ('19191919-1919-1919-1919-191919191919', 'test-workspace', 1000, 'success'::job_status, + '{"lock": "DEPS_RESULT"}'), + ('20202020-2020-2020-2020-202020202020', 'test-workspace', 1000, 'success'::job_status, + '{"lock": "WILDCARD_DEPS_RESULT"}'); + -- A token pairing an app scope with a run scope, as someone driving an app's components -- programmatically would build. `APP_INLINE_JOB` is an inline-script component run: no -- `jobs:run` scope can name its kind, so only the `apps:run` half puts it in reach. diff --git a/backend/tests/fixtures/schedule.sql b/backend/tests/fixtures/schedule.sql index 8d851369b0..999aa4770f 100644 --- a/backend/tests/fixtures/schedule.sql +++ b/backend/tests/fixtures/schedule.sql @@ -13,7 +13,7 @@ export async function main(fail: boolean = true) { '{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{"fail":{"default":true,"description":"","type":"boolean"}},"required":[],"type":"object"}', '', '', -'f/system/failing_script', 12349, 'deno', ''); +'f/system/failing_script', 12349, 'bun', E'{}\n//bun.lock\n'); INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES ( 'test-workspace', @@ -26,7 +26,7 @@ export async function main() { '{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{"path":{"default":null,"description":"","type":"string"},"schedule_path":{"default":null,"description":"","type":"string"},"error":{"default":null,"description":"","properties":{},"type":"object"}},"required":["path","schedule_path","error"],"type":"object"}', '', '', -'f/system/schedule_error_handler', 123410, 'deno', ''); +'f/system/schedule_error_handler', 123410, 'bun', E'{}\n//bun.lock\n'); INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES ( 'test-workspace', @@ -39,7 +39,7 @@ export async function main() { '{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{"path":{"default":null,"description":"","type":"string"},"schedule_path":{"default":null,"description":"","type":"string"},"previous_job_error":{"default":null,"description":"","type":"string"},"result":{"default":null,"description":"","type":"string"}},"required":["path","schedule_path","previous_job_error","result"],"type":"object"}', '', '', -'f/system/schedule_recovery_handler', 123411, 'deno', ''); +'f/system/schedule_recovery_handler', 123411, 'bun', E'{}\n//bun.lock\n'); INSERT INTO public.flow(workspace_id, summary, description, path, versions, schema, value, edited_by) VALUES ( 'test-workspace', diff --git a/backend/tests/flow_engine_parity.rs b/backend/tests/flow_engine_parity.rs index 7ea221c2e8..2d4f72f336 100644 --- a/backend/tests/flow_engine_parity.rs +++ b/backend/tests/flow_engine_parity.rs @@ -50,6 +50,7 @@ fn flow_module(id: &str, value: FlowModuleValue) -> FlowModule { apply_preprocessor: None, pass_flow_input_directly: None, debouncing: None, + job_token_scopes: None, } } @@ -84,7 +85,7 @@ async fn test_flow_linear_input_transforms(db: Pool) -> anyhow::Result modules: vec![ flow_module("a", FlowModuleValue::RawScript { input_transforms: [static_input("x", 10), static_input("y", 5)].into(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#" export function main(x: number, y: number) { return {sum: x + y, product: x * y, items: [1, 2, 3, 4, 5]}; @@ -104,7 +105,7 @@ export function main(x: number, y: number) { js_input("filtered", "results.a.items.filter(x => x > 2)"), js_input("from_flow_input", "flow_input.multiplier * results.a.sum"), ].into(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#" export function main(total: number, doubled_items: number[], filtered: number[], from_flow_input: number) { return {total, doubled_items, filtered, from_flow_input}; @@ -156,7 +157,7 @@ async fn test_flow_forloop_complex_expressions(db: Pool) -> anyhow::Re "a", FlowModuleValue::RawScript { input_transforms: Default::default(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#" export function main() { return { @@ -198,7 +199,7 @@ export function main() { js_input("index", "flow_input.iter.index"), ] .into(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#" export function main(user_name: string, user_score: number, bonus: number, index: number) { return {name: user_name, final_score: user_score + bonus, position: index}; @@ -260,7 +261,7 @@ async fn test_flow_branchone_conditions(db: Pool) -> anyhow::Result<() "a", FlowModuleValue::RawScript { input_transforms: Default::default(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#" export function main() { return {status: "premium", score: 95, items: [1, 2, 3]}; @@ -292,7 +293,7 @@ export function main() { js_input("score_from_a", "results.a.score"), ] .into(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#" export function main(discount: number, score_from_a: number) { return {branch: "premium_high", discount, score_from_a}; @@ -319,7 +320,7 @@ export function main(discount: number, score_from_a: number) { "premium_low", FlowModuleValue::RawScript { input_transforms: Default::default(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#" export function main() { return {branch: "premium_low", discount: 10}; @@ -343,7 +344,7 @@ export function main() { "default_branch", FlowModuleValue::RawScript { input_transforms: Default::default(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#" export function main() { return {branch: "default", discount: 0}; @@ -397,7 +398,7 @@ async fn test_flow_previous_result_aggregation(db: Pool) -> anyhow::Re "a", FlowModuleValue::RawScript { input_transforms: Default::default(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#" export function main() { return {value: 10, items: [1, 2, 3]}; @@ -423,7 +424,7 @@ export function main() { ), ] .into(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#" export function main(prev_value: number, prev_items_sum: number) { return {value: prev_value * 2, sum: prev_items_sum}; @@ -451,7 +452,7 @@ export function main(prev_value: number, prev_items_sum: number) { ), ] .into(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#" export function main(a_value: number, b_value: number, b_sum: number, combined: number) { return {a_value, b_value, b_sum, combined}; @@ -503,7 +504,7 @@ async fn test_flow_nested_complexity(db: Pool) -> anyhow::Result<()> { modules: vec![ flow_module("data", FlowModuleValue::RawScript { input_transforms: Default::default(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#" export function main() { return { @@ -540,7 +541,7 @@ export function main() { js_input("values", "results.data.base_values.map(v => v * flow_input.iter.value.multiplier)"), js_input("sum", "results.data.base_values.reduce((a, b) => a + b, 0) * flow_input.iter.value.multiplier"), ].into(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#" export function main(cat_name: string, multiplier: number, values: number[], sum: number) { return {category: cat_name, multiplier, computed_values: values, total: sum}; @@ -601,7 +602,7 @@ async fn test_flow_object_transformations(db: Pool) -> anyhow::Result< modules: vec![ flow_module("source", FlowModuleValue::RawScript { input_transforms: Default::default(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#" export function main() { return { @@ -632,7 +633,7 @@ export function main() { js_input("active_bonus", "results.source.config.activeBonus"), js_input("all_users", "results.source.users"), ].into(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#" export function main(admins: string[], active_count: number, admin_bonus: number, active_bonus: number, all_users: any[]) { // Compute user_summary and total_bonus in the script since complex expressions in input_transforms @@ -701,7 +702,7 @@ async fn test_flow_skip_if_expressions(db: Pool) -> anyhow::Result<()> "check", FlowModuleValue::RawScript { input_transforms: Default::default(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#" export function main() { return {should_skip: true, value: 100}; @@ -721,7 +722,7 @@ export function main() { "maybe_skipped", FlowModuleValue::RawScript { input_transforms: [js_input("input_val", "results.check.value * 2")].into(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#" export function main(input_val: number) { return {processed: input_val, was_run: true}; @@ -749,7 +750,7 @@ export function main(input_val: number) { js_input("prev", "previous_result"), ] .into(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#" export function main(check_val: number, prev: any) { return {check_val, previous: prev}; @@ -798,7 +799,7 @@ async fn test_flow_template_literals(db: Pool) -> anyhow::Result<()> { modules: vec![ flow_module("data", FlowModuleValue::RawScript { input_transforms: Default::default(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#" export function main() { return { @@ -824,7 +825,7 @@ export function main() { js_input("upper_name", "results.data.firstName.toUpperCase()"), js_input("items_formatted", "`Items: ${results.data.items.map(i => i.charAt(0).toUpperCase() + i.slice(1)).join(', ')}`"), ].into(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#" export function main(full_name: string, greeting: string, items_str: string, upper_name: string, items_formatted: string) { return {full_name, greeting, items_str, upper_name, items_formatted}; @@ -872,7 +873,7 @@ async fn test_flow_optional_chaining(db: Pool) -> anyhow::Result<()> { modules: vec![ flow_module("data", FlowModuleValue::RawScript { input_transforms: Default::default(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#" export function main() { return { @@ -904,7 +905,7 @@ export function main() { js_input("default_city", "results.data.partial_user?.address?.city ?? 'Unknown'"), js_input("default_country", "results.data.user?.address?.country ?? 'USA'"), ].into(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#" export function main(city: string, missing_city: any, null_user_name: any, default_city: string, default_country: string) { return {city, missing_city, null_user_name, default_city, default_country}; @@ -954,7 +955,7 @@ async fn test_flow_parallel_forloop(db: Pool) -> anyhow::Result<()> { "data", FlowModuleValue::RawScript { input_transforms: Default::default(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#" export function main() { return {items: [1, 2, 3, 4, 5]}; @@ -989,7 +990,7 @@ export function main() { js_input("cubed", "flow_input.iter.value.value ** 3"), ] .into(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#" export function main(original: number, squared: number, cubed: number) { return {original, squared, cubed}; @@ -1088,7 +1089,7 @@ async fn test_flow_env_access(db: Pool) -> anyhow::Result<()> { js_input("retry_count", "flow_env.CONFIG.retries"), js_input("has_auth", "flow_env.CONFIG.features.includes('auth')"), ].into(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#" export function main(env_name: string, is_debug: boolean, timeout_val: number, api_url: string, retry_count: number, has_auth: boolean) { return {env_name, is_debug, timeout_val, api_url, retry_count, has_auth}; @@ -1157,7 +1158,7 @@ async fn test_flow_input_and_env_combined(db: Pool) -> anyhow::Result< // Conditional based on both js_input("multiplier", "flow_env.ENV === 'production' ? flow_input.prod_mult : 1"), ].into(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#" export function main(effective_limit: number, env_prefix: string, is_prod: boolean, doubled_input: number, multiplier: number) { return {effective_limit, env_prefix, is_prod, doubled_input, final_value: doubled_input * multiplier}; @@ -1214,7 +1215,7 @@ async fn test_flow_results_optional_chaining(db: Pool) -> anyhow::Resu "a", FlowModuleValue::RawScript { input_transforms: Default::default(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#" export function main() { return { @@ -1286,7 +1287,7 @@ export function main() { ), ] .into(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#" export function main( user_name: string, @@ -1360,7 +1361,7 @@ async fn test_flow_large_integers(db: Pool) -> anyhow::Result<()> { "a", FlowModuleValue::RawScript { input_transforms: Default::default(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#" export function main() { return { @@ -1400,7 +1401,7 @@ export function main() { ), ] .into(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#" export function main( small: number, i32_max: number, over_i32: number, @@ -1458,7 +1459,7 @@ async fn test_flow_unicode_emoji(db: Pool) -> anyhow::Result<()> { modules: vec![ flow_module("a", FlowModuleValue::RawScript { input_transforms: Default::default(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#" export function main() { return { @@ -1488,7 +1489,7 @@ export function main() { js_input("mixed_upper", "results.a.mixed.toUpperCase()"), js_input("template", "`Welcome: ${results.a.greeting}`"), ].into(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#" export function main( greeting: string, greeting_len: number, simple_str: string, has_world: boolean, @@ -1543,7 +1544,7 @@ async fn test_flow_complex_array_operations(db: Pool) -> anyhow::Resul modules: vec![ flow_module("a", FlowModuleValue::RawScript { input_transforms: Default::default(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#" export function main() { return { @@ -1584,7 +1585,7 @@ export function main() { js_input("first_three", "results.a.numbers.slice(0, 3)"), js_input("last_two", "results.a.numbers.slice(-2)"), ].into(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#" export function main( sorted_asc: number[], sorted_desc: number[], active_names: string[], @@ -1650,7 +1651,7 @@ async fn test_flow_multiline_expressions(db: Pool) -> anyhow::Result<( modules: vec![ flow_module("a", FlowModuleValue::RawScript { input_transforms: Default::default(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#" export function main() { return { @@ -1709,7 +1710,7 @@ export function main() { } "#), ].into(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#" export function main( subtotal: number, discounted_total: number, @@ -1767,7 +1768,7 @@ async fn test_flow_spread_with_results(db: Pool) -> anyhow::Result<()> modules: vec![ flow_module("a", FlowModuleValue::RawScript { input_transforms: Default::default(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#" export function main() { return { @@ -1802,7 +1803,7 @@ export function main() { // Combining multiple spreads js_input("combined", "{config: {...results.a.config}, tags: [...results.a.tags], source: 'flow'}"), ].into(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#" export function main( merged_config: any, all_tags: string[], full_user: any, @@ -1868,7 +1869,7 @@ async fn test_flow_nested_forloop_results_access(db: Pool) -> anyhow:: "a", FlowModuleValue::RawScript { input_transforms: Default::default(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#" export function main() { return { @@ -1907,7 +1908,7 @@ export function main() { js_input("multiplier", "results.a.multiplier"), // Access outer step from inside for-loop ] .into(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#" export function main(category: string, multiplier: number) { return { @@ -1938,7 +1939,7 @@ export function main(category: string, multiplier: number) { js_input("original_mult", "results.a.multiplier"), // Access outer step ] .into(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#" export function main(items: any[], category: string, original_mult: number) { return { @@ -2012,7 +2013,7 @@ async fn test_flow_results_non_existent_step(db: Pool) -> anyhow::Resu "a", FlowModuleValue::RawScript { input_transforms: Default::default(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#" export function main() { return { value: 42 }; @@ -2049,7 +2050,7 @@ export function main() { ), ] .into(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#" export function main( existing: number, @@ -2125,7 +2126,7 @@ async fn test_flow_env_skip_if_inside_branchone(db: Pool) -> anyhow::R "inner", FlowModuleValue::RawScript { input_transforms: Default::default(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#" export function main() { return {ran: true}; @@ -2155,7 +2156,7 @@ export function main() { "branch_marker", FlowModuleValue::RawScript { input_transforms: Default::default(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#" export function main() { return {marker: "branch-marker"}; @@ -2192,7 +2193,7 @@ export function main() { "after", FlowModuleValue::RawScript { input_transforms: [js_input("prev", "previous_result")].into(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#" export function main(prev: any) { return {prev}; @@ -2257,7 +2258,7 @@ async fn test_flow_env_skip_if_nested_branchone(db: Pool) -> anyhow::R "leaf", FlowModuleValue::RawScript { input_transforms: Default::default(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#" export function main() { return {ran: true}; @@ -2281,7 +2282,7 @@ export function main() { "inner_marker", FlowModuleValue::RawScript { input_transforms: Default::default(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#" export function main() { return {marker: "inner"}; @@ -2336,7 +2337,7 @@ export function main() { "after", FlowModuleValue::RawScript { input_transforms: [js_input("prev", "previous_result")].into(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#" export function main(prev: any) { return {prev}; @@ -2413,7 +2414,7 @@ async fn test_flow_env_complex_input_transform_in_branch(db: Pool) -> "Math.min(flow_env.LIMIT, 10) + flow_env.OFFSET", )] .into(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#" export function main(value: number) { return {value}; @@ -2486,7 +2487,7 @@ async fn test_flow_env_skip_if_in_parallel_forloop(db: Pool) -> anyhow "marker", FlowModuleValue::RawScript { input_transforms: [js_input("i", "flow_input.iter.value")].into(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#" export function main(i: number) { return {marker: i}; @@ -2507,7 +2508,7 @@ export function main(i: number) { "leaf", FlowModuleValue::RawScript { input_transforms: Default::default(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#" export function main() { return {ran: true}; @@ -2593,7 +2594,7 @@ async fn test_flow_env_imported_flow_uses_own_env(db: Pool) -> anyhow: "id": "leaf", "value": { "type": "rawscript", - "language": "deno", + "language": "bun", "content": "export function main(key: string) { return {key}; }", "input_transforms": { "key": { "type": "javascript", "expr": "flow_env.KEY" } @@ -2678,7 +2679,7 @@ async fn test_flow_env_imported_flow_inherits_when_unset(db: Pool) -> "id": "marker", "value": { "type": "rawscript", - "language": "deno", + "language": "bun", "content": "export function main() { return {marker: \"from-imported\"}; }", "input_transforms": {} } @@ -2687,7 +2688,7 @@ async fn test_flow_env_imported_flow_inherits_when_unset(db: Pool) -> "id": "leaf", "value": { "type": "rawscript", - "language": "deno", + "language": "bun", "content": "export function main() { return {ran: true}; }", "input_transforms": {} }, @@ -2790,7 +2791,7 @@ async fn test_flow_env_imported_flow_with_nested_branch(db: Pool) -> a "id": "marker", "value": { "type": "rawscript", - "language": "deno", + "language": "bun", "content": "export function main() { return {marker: \"from-imported-branch\"}; }", "input_transforms": {} } @@ -2799,7 +2800,7 @@ async fn test_flow_env_imported_flow_with_nested_branch(db: Pool) -> a "id": "leaf", "value": { "type": "rawscript", - "language": "deno", + "language": "bun", "content": "export function main() { return {ran: true}; }", "input_transforms": {} }, @@ -2931,13 +2932,13 @@ async fn test_flow_env_marks_sub_flows_only_without_ancestor_env( "parallel": true, "modules": [{ "id": "leaf", "value": { "type": "rawscript", - "language": "deno", + "language": "bun", "content": "export function main(key: any) { return key ?? null; }", "input_transforms": { "key": { "type": "javascript", "expr": "flow_env.KEY" } } }}, { "id": "tail", "value": { // A single simple step is inlined rather than stored as a flow node. "type": "rawscript", - "language": "deno", + "language": "bun", "content": "export function main(prev: any) { return prev; }", "input_transforms": { "prev": { "type": "javascript", "expr": "results.leaf" } } }}] @@ -2974,6 +2975,7 @@ async fn test_flow_env_marks_sub_flows_only_without_ancestor_env( apply_preprocessor: false, version, labels: None, + job_token_scopes: None, }) .run_until_complete(&db, false, port) .await; @@ -3022,7 +3024,7 @@ async fn test_flow_env_in_stop_after_if(db: Pool) -> anyhow::Result<() "first", FlowModuleValue::RawScript { input_transforms: Default::default(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#" export function main() { return {stage: "first"}; @@ -3050,7 +3052,7 @@ export function main() { "second", FlowModuleValue::RawScript { input_transforms: Default::default(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#" export function main() { return {stage: "second"}; @@ -3108,7 +3110,7 @@ async fn test_stop_after_if_error_include_result(db: Pool) -> anyhow:: "step", FlowModuleValue::RawScript { input_transforms: Default::default(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#" export function main() { return { userErrors: ["email taken"], ok: false }; @@ -3198,7 +3200,7 @@ async fn test_flow_env_in_retry_if(db: Pool) -> anyhow::Result<()> { "fails", FlowModuleValue::RawScript { input_transforms: Default::default(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#" export function main() { throw new Error("nope"); @@ -3272,7 +3274,7 @@ async fn test_flow_env_in_stop_after_all_iters_if(db: Pool) -> anyhow: "iter_step", FlowModuleValue::RawScript { input_transforms: [js_input("i", "flow_input.iter.value")].into(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#" export function main(i: number) { return {iter: i}; @@ -3314,7 +3316,7 @@ export function main(i: number) { "after", FlowModuleValue::RawScript { input_transforms: Default::default(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#" export function main() { return {stage: "after-loop"}; @@ -3373,7 +3375,7 @@ async fn test_stop_after_all_iters_if_error_includes_result( "iter_step", FlowModuleValue::RawScript { input_transforms: [js_input("i", "flow_input.iter.value")].into(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#" export function main(i: number) { return { iter: i }; @@ -3451,8 +3453,8 @@ async fn test_flow_expr_step_tag(db: Pool) -> anyhow::Result<()> { id, FlowModuleValue::RawScript { input_transforms: Default::default(), - language: ScriptLang::Deno, - content: "export function main() { return { lang: 'bun' } }".to_string(), + language: ScriptLang::Bun, + content: "export function main() { return { lang: 'python3' } }".to_string(), path: None, lock: None, tag: tag.map(str::to_string), @@ -3482,7 +3484,7 @@ async fn test_flow_expr_step_tag(db: Pool) -> anyhow::Result<()> { .bind(job.id) .fetch_one(&db) .await?; - assert_eq!(b_tag, "bun"); + assert_eq!(b_tag, "python3"); assert!(!job.success); let result = job.json_result().unwrap(); diff --git a/backend/tests/instance_config.rs b/backend/tests/instance_config.rs index 52ebbee7e7..24937a3ce5 100644 --- a/backend/tests/instance_config.rs +++ b/backend/tests/instance_config.rs @@ -237,7 +237,7 @@ async fn test_diff_global_settings_empty_string_routes_to_delete(db: Pool) { ..Default::default() }; - apply_settings_diff(&db, &diff).await.unwrap(); + apply_settings_diff(&db, &diff, &()).await.unwrap(); assert_eq!( get_global_setting(&db, "key_a").await, @@ -468,7 +468,7 @@ async fn test_apply_settings_diff_deletes_only(db: Pool) { ..Default::default() }; - apply_settings_diff(&db, &diff).await.unwrap(); + apply_settings_diff(&db, &diff, &()).await.unwrap(); assert!(get_global_setting(&db, "to_delete_1").await.is_none()); assert!(get_global_setting(&db, "to_delete_2").await.is_none()); @@ -489,7 +489,7 @@ async fn test_apply_settings_diff_upserts_and_deletes(db: Pool) { ..Default::default() }; - apply_settings_diff(&db, &diff).await.unwrap(); + apply_settings_diff(&db, &diff, &()).await.unwrap(); assert!(get_global_setting(&db, "old_key").await.is_none()); assert_eq!( @@ -503,7 +503,7 @@ async fn test_apply_settings_diff_empty_noop(db: Pool) { insert_global_setting(&db, "preexisting", serde_json::json!("value")).await; let diff = SettingsDiff::default(); - apply_settings_diff(&db, &diff).await.unwrap(); + apply_settings_diff(&db, &diff, &()).await.unwrap(); assert_eq!( get_global_setting(&db, "preexisting").await, @@ -525,7 +525,7 @@ async fn test_apply_settings_diff_upsert_overwrites(db: Pool) { ..Default::default() }; - apply_settings_diff(&db, &diff).await.unwrap(); + apply_settings_diff(&db, &diff, &()).await.unwrap(); assert_eq!( get_global_setting(&db, "overwrite_me").await, @@ -553,7 +553,7 @@ async fn test_apply_settings_diff_complex_json(db: Pool) { ..Default::default() }; - apply_settings_diff(&db, &diff).await.unwrap(); + apply_settings_diff(&db, &diff, &()).await.unwrap(); let stored = get_global_setting(&db, "complex_setting").await.unwrap(); assert_eq!(stored, complex_value); @@ -568,7 +568,7 @@ async fn test_apply_settings_diff_delete_nonexistent_is_noop(db: Pool) }; // Should not error - apply_settings_diff(&db, &diff).await.unwrap(); + apply_settings_diff(&db, &diff, &()).await.unwrap(); } // ======================================================================== @@ -589,7 +589,7 @@ async fn test_apply_configs_diff_upserts_with_prefix(db: Pool) { deletes: vec![], }; - apply_configs_diff(&db, &diff).await.unwrap(); + apply_configs_diff(&db, &diff, &()).await.unwrap(); let stored = get_config(&db, "worker__mygroup").await.unwrap(); assert_eq!(stored["init_bash"], "echo hi"); @@ -601,7 +601,7 @@ async fn test_apply_configs_diff_deletes_with_prefix(db: Pool) { let diff = ConfigsDiff { upserts: BTreeMap::new(), deletes: vec!["to_remove".to_string()] }; - apply_configs_diff(&db, &diff).await.unwrap(); + apply_configs_diff(&db, &diff, &()).await.unwrap(); assert!(get_config(&db, "worker__to_remove").await.is_none()); } @@ -619,7 +619,7 @@ async fn test_apply_configs_diff_upsert_overwrites(db: Pool) { deletes: vec![], }; - apply_configs_diff(&db, &diff).await.unwrap(); + apply_configs_diff(&db, &diff, &()).await.unwrap(); let stored = get_config(&db, "worker__grp").await.unwrap(); assert_eq!(stored, serde_json::json!({"new": true})); @@ -630,7 +630,7 @@ async fn test_apply_configs_diff_empty_noop(db: Pool) { insert_config(&db, "worker__keep", serde_json::json!({"keep": true})).await; let diff = ConfigsDiff::default(); - apply_configs_diff(&db, &diff).await.unwrap(); + apply_configs_diff(&db, &diff, &()).await.unwrap(); assert!(get_config(&db, "worker__keep").await.is_some()); } @@ -641,7 +641,7 @@ async fn test_apply_configs_diff_does_not_touch_non_worker_configs(db: Pool) { assert!(diff.upserts.contains_key("expose_metrics")); assert!(diff.deletes.is_empty()); - apply_settings_diff(&db, &diff).await.unwrap(); + apply_settings_diff(&db, &diff, &()).await.unwrap(); // Step 5: Read back and verify let config2 = InstanceConfig::from_db(&db).await.unwrap(); @@ -729,7 +729,7 @@ async fn test_roundtrip_replace_mode_deletes(db: Pool) { assert!(diff.deletes.contains(&"retention_period_secs".to_string())); assert!(diff.deletes.contains(&"expose_metrics".to_string())); - apply_settings_diff(&db, &diff).await.unwrap(); + apply_settings_diff(&db, &diff, &()).await.unwrap(); assert!(get_global_setting(&db, "retention_period_secs") .await @@ -782,7 +782,7 @@ async fn test_roundtrip_worker_configs(db: Pool) { assert!(diff.upserts.contains_key("gpu")); // new assert_eq!(diff.deletes, vec!["legacy".to_string()]); - apply_configs_diff(&db, &diff).await.unwrap(); + apply_configs_diff(&db, &diff, &()).await.unwrap(); // Verify let config2 = InstanceConfig::from_db(&db).await.unwrap(); @@ -820,7 +820,7 @@ async fn test_roundtrip_to_settings_map_from_db_consistency(db: Pool) let diff = SettingsDiff { upserts: map.into_iter().collect(), deletes: vec![], ..Default::default() }; - apply_settings_diff(&db, &diff).await.unwrap(); + apply_settings_diff(&db, &diff, &()).await.unwrap(); let config = InstanceConfig::from_db(&db).await.unwrap(); assert_eq!(config.global_settings.base_url, original.base_url); @@ -868,8 +868,8 @@ async fn test_idempotent_apply(db: Pool) { }; // Apply twice - apply_settings_diff(&db, &diff).await.unwrap(); - apply_settings_diff(&db, &diff).await.unwrap(); + apply_settings_diff(&db, &diff, &()).await.unwrap(); + apply_settings_diff(&db, &diff, &()).await.unwrap(); assert_eq!( get_global_setting(&db, "idem_key").await, @@ -1069,7 +1069,7 @@ async fn test_diff_apply_only_touches_changed_rows(db: Pool) { assert!(!diff.upserts.contains_key("unchanged_1")); assert!(!diff.upserts.contains_key("unchanged_2")); - apply_settings_diff(&db, &diff).await.unwrap(); + apply_settings_diff(&db, &diff, &()).await.unwrap(); // Verify all 4 settings present assert_eq!(count_global_settings(&db).await, 4); @@ -1148,7 +1148,7 @@ async fn test_replace_mode_protects_settings_in_integration(db: Pool) // But normal_setting should be deleted assert!(diff.deletes.contains(&"normal_setting".to_string())); - apply_settings_diff(&db, &diff).await.unwrap(); + apply_settings_diff(&db, &diff, &()).await.unwrap(); // Verify protected settings survived assert!(get_global_setting(&db, "ducklake_settings").await.is_some()); @@ -1196,7 +1196,7 @@ async fn test_jwt_secret_roundtrip_as_string_or_secret_ref(db: Pool) { let diff = SettingsDiff { upserts: map.into_iter().collect(), deletes: vec![], ..Default::default() }; - apply_settings_diff(&db, &diff).await.unwrap(); + apply_settings_diff(&db, &diff, &()).await.unwrap(); // Read back from DB — jwt_secret should survive the roundtrip let config = InstanceConfig::from_db(&db).await.unwrap(); @@ -1239,7 +1239,7 @@ async fn test_rsa_keys_roundtrip_via_extra(db: Pool) { let map = settings.to_settings_map(); let diff = SettingsDiff { upserts: map.into_iter().collect(), deletes: vec![], ..Default::default() }; - apply_settings_diff(&db, &diff).await.unwrap(); + apply_settings_diff(&db, &diff, &()).await.unwrap(); // Read back from DB let config = InstanceConfig::from_db(&db).await.unwrap(); @@ -1296,7 +1296,7 @@ async fn test_replace_mode_protects_jwt_secret_and_rsa_keys(db: Pool) "normal_setting should be deleted" ); - apply_settings_diff(&db, &diff).await.unwrap(); + apply_settings_diff(&db, &diff, &()).await.unwrap(); // Both protected settings survive assert_eq!( @@ -1371,7 +1371,7 @@ async fn test_alert_config_in_global_settings_roundtrip(db: Pool) { "alert_config change should be detected in diff" ); - apply_settings_diff(&db, &diff).await.unwrap(); + apply_settings_diff(&db, &diff, &()).await.unwrap(); // Re-read and verify the update let config2 = InstanceConfig::from_db(&db).await.unwrap(); @@ -1455,6 +1455,7 @@ async fn declarative_sync_rejects_an_unusable_webhook_base_url(db: Pool &db, &BTreeMap::new(), &desired, + &(), ) .await .expect_err("a javascript: banner link must fail the sync"); @@ -1545,6 +1548,7 @@ async fn declarative_sync_rejects_a_malformed_max_token_expiration(db: Pool) { &db, &BTreeMap::new(), &desired, + &(), ) .await .expect_err("a non-hex accent color must fail the sync"); @@ -1681,7 +1691,109 @@ async fn declarative_sync_rejects_a_non_hex_accent_color(db: Pool) { &db, &BTreeMap::new(), &desired, + &(), ) .await .expect("a hex accent color must sync"); } + +/// A declarative sync has no user request behind it, so its observer is the only thing +/// that audits it: one entry per setting or worker config it actually writes, values +/// recorded under the API path's rule, and none for what it leaves as is. +#[cfg(all(feature = "enterprise", feature = "private"))] +#[sqlx::test(fixtures("base"))] +async fn test_declarative_sync_audits_only_what_it_writes(db: Pool) { + use windmill_audit::instance_config::{ + declarative_sync_author, DeclarativeSource, InstanceConfigAuditor, + }; + + insert_global_setting(&db, "retention_period_secs", serde_json::json!(100)).await; + insert_global_setting(&db, "expose_metrics", serde_json::json!(true)).await; + insert_global_setting(&db, "scim_token", serde_json::json!("old-scim-secret")).await; + insert_global_setting( + &db, + "npm_config_registry", + serde_json::json!("https://u:p@npm"), + ) + .await; + insert_config( + &db, + "worker__kept", + serde_json::json!({"worker_tags": ["a"]}), + ) + .await; + insert_config( + &db, + "worker__dropped", + serde_json::json!({"worker_tags": ["b"]}), + ) + .await; + + let config: windmill_common::instance_config::InstanceConfig = + serde_json::from_value(serde_json::json!({ + "global_settings": { + "retention_period_secs": 200, + "expose_metrics": true, + "scim_token": "new-scim-secret", + }, + "worker_configs": { + "kept": {"worker_tags": ["a"]}, + "added": {"init_bash": "export TOKEN=init-secret"}, + }, + })) + .unwrap(); + let author = declarative_sync_author(); + config + .sync_to_db( + &db, + &InstanceConfigAuditor { + db: &db, + author: &author, + source: Some(DeclarativeSource::SyncConfig), + }, + ) + .await + .unwrap(); + + let entries: Vec<(String, String, serde_json::Value)> = sqlx::query_as( + "SELECT operation, resource, parameters FROM audit_partitioned \ + WHERE workspace_id = 'global' AND username = $1 AND resource = ANY($2) \ + ORDER BY resource", + ) + .bind(&author.username) + .bind(vec![ + "retention_period_secs", + "expose_metrics", + "scim_token", + "npm_config_registry", + "worker__kept", + "worker__dropped", + "worker__added", + ]) + .fetch_all(&db) + .await + .unwrap(); + + let source = serde_json::json!({"source": "sync-config"}); + let entry = |operation: &str, resource: &str, parameters: &serde_json::Value| { + ( + operation.to_string(), + resource.to_string(), + parameters.clone(), + ) + }; + assert_eq!( + entries, + vec![ + entry("global_settings.delete", "npm_config_registry", &source), + entry( + "global_settings.update", + "retention_period_secs", + &serde_json::json!({"source": "sync-config", "value": "200"}), + ), + entry("global_settings.update", "scim_token", &source), + entry("worker_config.update", "worker__added", &source), + entry("worker_config.delete", "worker__dropped", &source), + ] + ); +} diff --git a/backend/tests/job_payload.rs b/backend/tests/job_payload.rs index 1a973f6fe6..c40feb1650 100644 --- a/backend/tests/job_payload.rs +++ b/backend/tests/job_payload.rs @@ -54,9 +54,10 @@ mod job_payload { cache_ttl: None, cache_ignore_s3_path: None, dedicated_worker: None, - language: ScriptLang::Deno, + language: ScriptLang::Bun, priority: None, apply_preprocessor: false, + job_token_scopes: None, }) .arg("world", json!("foo")) .run_until_complete(&db, false, port) @@ -84,7 +85,7 @@ mod job_payload { cache_ttl: None, cache_ignore_s3_path: None, dedicated_worker: None, - language: ScriptLang::Deno, + language: ScriptLang::Bun, priority: None, apply_preprocessor: true, concurrency_settings: @@ -92,6 +93,7 @@ mod job_payload { debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), labels: None, + job_token_scopes: None, }) .run_until_complete_with(db, false, port, |id| async move { let job = sqlx::query!("SELECT preprocessed FROM v2_job WHERE id = $1", id) @@ -163,7 +165,7 @@ mod job_payload { let test = || async { let result = RunJob::from(JobPayload::FlowScript { id: flow_scripts[0], - language: ScriptLang::Deno, + language: ScriptLang::Bun, concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default(), cache_ttl: None, @@ -183,7 +185,7 @@ mod job_payload { let test = || async { let result = RunJob::from(JobPayload::FlowScript { id: flow_scripts[1], - language: ScriptLang::Deno, + language: ScriptLang::Bun, concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default(), cache_ttl: None, @@ -261,7 +263,7 @@ mod job_payload { let result = RunJob::from(JobPayload::Dependencies { path: "f/system/hello".to_string(), hash: ScriptHash(123412), - language: ScriptLang::Deno, + language: ScriptLang::Bun, debouncing_settings: Default::default(), dedicated_worker: None, }) @@ -349,7 +351,7 @@ mod job_payload { console.log(greet) return greet }"#, - "language": "deno", + "language": "bun", "input_transforms": { "world": { "type": "javascript", "expr": "flow_input.world" } } @@ -405,7 +407,7 @@ mod job_payload { return greet }"# .into(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, }) .run_until_complete(&db, false, port) .await @@ -414,7 +416,10 @@ mod job_payload { assert_eq!( result, - json!({ "lock": "", "status": "Successful lock file generation" }) + json!({ + "lock": "{\n \"dependencies\": {}\n}\n//bun.lock\n", + "status": "Successful lock file generation" + }) ); }; test_for_versions(VERSION_FLAGS.iter().copied(), test).await; @@ -435,6 +440,7 @@ mod job_payload { apply_preprocessor: false, version: 1443253234253454, labels: None, + job_token_scopes: None, }) .run_until_complete(&db, false, port) .await @@ -485,6 +491,7 @@ mod job_payload { apply_preprocessor: true, version: 1443253234253456, labels: None, + job_token_scopes: None, }) .run_until_complete_with(db, false, port, |id| async move { let job = sqlx::query!("SELECT preprocessed FROM v2_job WHERE id = $1", id) @@ -556,6 +563,7 @@ mod job_payload { apply_preprocessor: true, version: 1443253234253454, labels: None, + job_token_scopes: None, }) .run_until_complete(&db, false, port) .await @@ -621,7 +629,7 @@ mod job_payload { console.log(greet) return greet }"#, - "language": "deno", + "language": "bun", "input_transforms": { "world": { "type": "javascript", "expr": "flow_input.world" } } @@ -668,7 +676,7 @@ mod job_payload { "content": r#"export function main(world: string) { return `Hello ${world}!`; }"#, - "language": "deno", + "language": "bun", "input_transforms": { "world": { "type": "javascript", "expr": "flow_input.world" } } @@ -680,7 +688,7 @@ mod job_payload { "content": r#"export function main(world: string, a: string) { return `${a} ${world}!`; }"#, - "language": "deno", + "language": "bun", "input_transforms": { "world": { "type": "javascript", "expr": "flow_input.world" }, "a": { "type": "javascript", "expr": "results.a" } @@ -699,7 +707,7 @@ mod job_payload { "x": { "type": "javascript", "expr": "flow_input.iter.value" } }, "type": "rawscript", - "language": "deno", + "language": "bun", "content": r#"export function main(world: string, b: string, x: string) { return `${x}: ${b} ${world}!`; }"#, @@ -852,7 +860,7 @@ mod job_payload { "id": "a", "value": { "type": "rawscript", - "language": "deno", + "language": "bun", "input_transforms": { "world": { "type": "javascript", "expr": "flow_input.world" } }, @@ -869,7 +877,7 @@ mod job_payload { "id": "inner_first", "value": { "type": "rawscript", - "language": "deno", + "language": "bun", "input_transforms": { "world": { "type": "javascript", "expr": "flow_input.world" } }, @@ -879,7 +887,7 @@ mod job_payload { "id": "inner_second", "value": { "type": "rawscript", - "language": "deno", + "language": "bun", "input_transforms": { "world": { "type": "javascript", "expr": "flow_input.world" }, "first": { "type": "javascript", "expr": "results.inner_first" } @@ -1040,7 +1048,7 @@ mod job_payload { "id": "inner", "value": { "type": "rawscript", - "language": "deno", + "language": "bun", "input_transforms": { "iter_val": { "type": "javascript", "expr": "flow_input.iter.value" }, "tag": { "type": "javascript", "expr": "flow_input.tag" } @@ -1150,6 +1158,7 @@ mod job_payload { apply_preprocessor: true, version: 1443253234253454, labels: None, + job_token_scopes: None, }) .run_until_complete(db, false, port) .await; @@ -1238,7 +1247,7 @@ mod job_payload { "id": "leaf", "value": { "type": "rawscript", - "language": "deno", + "language": "bun", "input_transforms": { "tag": { "type": "javascript", "expr": "flow_input.tag" }, "iv": { "type": "javascript", "expr": "flow_input.iter.value" } @@ -1382,7 +1391,7 @@ mod job_payload { "id": "leaf", "value": { "type": "rawscript", - "language": "deno", + "language": "bun", "input_transforms": { "tag": { "type": "javascript", "expr": "flow_input.tag" } }, @@ -1625,6 +1634,7 @@ mod job_payload { debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), labels: None, + job_token_scopes: None, }) .arg("foo", json!("hello")) .arg("bar", json!("world")) @@ -1676,6 +1686,7 @@ mod job_payload { debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), labels: None, + job_token_scopes: None, }) .arg("foo", json!("hello")) .arg("bar", json!("world")) @@ -1727,6 +1738,7 @@ mod job_payload { debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), labels: None, + job_token_scopes: None, }) .arg("foo", json!("hello")) .arg("bar", json!("world")) @@ -1778,6 +1790,7 @@ mod job_payload { debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), labels: None, + job_token_scopes: None, }) .arg("foo", json!("hello")) .arg("bar", json!("world")) @@ -1831,6 +1844,7 @@ mod job_payload { debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), labels: None, + job_token_scopes: None, }) .arg("foo", json!("hello")) .arg("bar", json!("world")) diff --git a/backend/tests/job_provenance.rs b/backend/tests/job_provenance.rs index 42ccf829e6..4eda4e5e17 100644 --- a/backend/tests/job_provenance.rs +++ b/backend/tests/job_provenance.rs @@ -150,3 +150,104 @@ async fn flow_steps_are_latest_only_under_a_current_unrestarted_flow(db: Pool) { + use serde_json::{json, Value}; + use windmill_common::job_provenance::RunAsType; + + let v: Value = + serde_json::from_str(include_str!("../../cli/test/fixtures/item_digest_vectors.json")) + .unwrap(); + let (script, flow) = (&v["script"], &v["flow"]); + + sqlx::query( + "INSERT INTO script (workspace_id, hash, path, content, lock, modules, language, kind, created_by, schema, summary, description) + VALUES ('test-workspace', 777, $1, $2, $3, $4, 'bun', 'script', 'test-user', '{}', '', '')", + ) + .bind(script["path"].as_str()) + .bind(script["content"].as_str()) + .bind(script["lock"].as_str()) + .bind(&script["modules"]) + .execute(&db) + .await + .unwrap(); + + // Version 889 holds step `a` by reference, which no checkout can reproduce. + let step = &flow["value"]["modules"][0]["value"]; + let mut by_ref = flow["value"].clone(); + by_ref["modules"][0]["value"] = json!({ + "type": "flowscript", "id": 999, "language": step["language"], + "input_transforms": step["input_transforms"], + }); + sqlx::query( + "INSERT INTO flow (workspace_id, path, summary, description, value, edited_by, versions) + VALUES ('test-workspace', $1, '', '', $2, 'test-user', ARRAY[888::bigint, 889::bigint])", + ) + .bind(flow["path"].as_str()) + .bind(&flow["value"]) + .execute(&db) + .await + .unwrap(); + for (id, value) in [(888, &flow["value"]), (889, &by_ref)] { + sqlx::query( + "INSERT INTO flow_version (id, workspace_id, path, value, schema, created_by) + VALUES ($1, 'test-workspace', $2, $3, '{}', 'test-user')", + ) + .bind(id as i64) + .bind(flow["path"].as_str()) + .bind(value) + .execute(&db) + .await + .unwrap(); + } + sqlx::query( + "INSERT INTO flow_node (id, workspace_id, path, code, lock, hash_v2) + VALUES (999, 'test-workspace', $1, $2, $3, 'h')", + ) + .bind(flow["path"].as_str()) + .bind(step["content"].as_str()) + .bind(step["lock"].as_str()) + .execute(&db) + .await + .unwrap(); + + sqlx::query( + "INSERT INTO v2_job (id, workspace_id, kind, runnable_path, runnable_id, parent_job, trigger, trigger_kind, tag, created_by, permissioned_as, permissioned_as_email) + VALUES + ('3bb0c0de-0000-4000-8000-000000000201', 'test-workspace', 'script', $1, 777, NULL, NULL, NULL, 'bun', 'test-user', 'g/all', 'group-all@windmill.dev'), + ('3bb0c0de-0000-4000-8000-000000000202', 'test-workspace', 'flow', $2, 888, NULL, 'f/digest/nightly', 'schedule', 'flow', 'test-user', 'u/test-user', 'test@windmill.dev'), + ('3bb0c0de-0000-4000-8000-000000000203', 'test-workspace', 'flowscript', $2 || '/a', 999, '3bb0c0de-0000-4000-8000-000000000202', NULL, NULL, 'gpu', 'test-user', 'u/test-user', 'test@windmill.dev'), + ('3bb0c0de-0000-4000-8000-000000000204', 'test-workspace', 'flow', $2, 889, NULL, NULL, NULL, 'flow', 'test-user', 'u/test-user', 'test@windmill.dev')", + ) + .bind(script["path"].as_str()) + .bind(flow["path"].as_str()) + .execute(&db) + .await + .unwrap(); + for q in [ + "INSERT INTO worker_ping (worker, worker_instance, worker_group) VALUES ('wk-gpu', 'wk', 'gpu-group')", + "INSERT INTO v2_job_queue (id, workspace_id, scheduled_for, running, worker, tag) + VALUES ('3bb0c0de-0000-4000-8000-000000000203', 'test-workspace', now(), true, 'wk-gpu', 'gpu')", + ] { + sqlx::query(q).execute(&db).await.unwrap(); + } + + let s = provenance(&db, "3bb0c0de-0000-4000-8000-000000000201").await; + assert_eq!(s.digest.as_deref(), script["digest"].as_str()); + assert_eq!(s.root_digest, s.digest); + assert_eq!(s.run_as_type, RunAsType::Group); + assert_eq!(s.worker_group, None); + + let step = provenance(&db, "3bb0c0de-0000-4000-8000-000000000203").await; + assert_eq!(step.digest.as_deref(), flow["steps"]["a"].as_str()); + assert_eq!(step.root_digest.as_deref(), flow["digest"].as_str()); + assert_eq!(step.root_trigger.as_deref(), Some("f/digest/nightly")); + assert_eq!(step.tag, "gpu"); + assert_eq!(step.worker_group.as_deref(), Some("gpu-group")); + assert_eq!(step.run_as_type, RunAsType::User); + + assert_eq!(provenance(&db, "3bb0c0de-0000-4000-8000-000000000204").await.digest, None); +} diff --git a/backend/tests/job_token_scopes.rs b/backend/tests/job_token_scopes.rs new file mode 100644 index 0000000000..f805168bfa --- /dev/null +++ b/backend/tests/job_token_scopes.rs @@ -0,0 +1,431 @@ +//! A job token restricted by `job_token_scopes` reaches only what its scopes allow, plus the +//! runtime routes about its own job, and no job it creates holds a wider token than it. + +use reqwest::StatusCode; +use serde_json::json; +use sqlx::{Pool, Postgres}; +use uuid::Uuid; +use windmill_common::jobs::JobPayload; +use windmill_test_utils::*; + +const OIDC_JOB: &str = "b0000000-0000-0000-0000-000000000001"; +const RUN_JOB: &str = "b0000000-0000-0000-0000-000000000002"; +const FLOW_JOB: &str = "b0000000-0000-0000-0000-000000000003"; + +async fn insert_job( + db: &Pool, + id: &str, + parent: Option<&str>, + scopes: &[&str], +) -> anyhow::Result<()> { + insert_job_with(db, id, parent, Some(scopes)).await +} + +async fn insert_job_with( + db: &Pool, + id: &str, + parent: Option<&str>, + scopes: Option<&[&str]>, +) -> anyhow::Result<()> { + let id = Uuid::parse_str(id)?; + sqlx::query( + "INSERT INTO v2_job (id, workspace_id, created_by, permissioned_as, permissioned_as_email, + kind, script_lang, runnable_path, tag, parent_job) + VALUES ($1, 'test-workspace', 'test-user-3', 'u/test-user-3', 'test3@windmill.dev', + 'script', 'deno', 'u/test-user-3/agent', 'deno', $2)", + ) + .bind(id) + .bind(parent.map(Uuid::parse_str).transpose()?) + .execute(db) + .await?; + sqlx::query( + "INSERT INTO job_perms (job_id, email, username, is_admin, is_operator, folders, groups, + workspace_id, job_token_scopes) + VALUES ($1, 'test3@windmill.dev', 'test-user-3', false, false, '{}', '{}', + 'test-workspace', $2)", + ) + .bind(id) + .bind(scopes) + .execute(db) + .await?; + Ok(()) +} + +async fn job_token(db: &Pool, id: &str) -> anyhow::Result { + Ok(windmill_common::auth::create_token_for_owner( + db, + "test-workspace", + "u/test-user-3", + "ephemeral-script", + 300, + "test3@windmill.dev", + &Uuid::parse_str(id)?, + None, + None, + ) + .await?) +} + +async fn insert_script( + db: &Pool, + path: &str, + hash: i64, + scopes: Option<&[&str]>, +) -> anyhow::Result<()> { + sqlx::query( + "INSERT INTO script (workspace_id, created_by, content, schema, summary, description, + path, hash, language, lock, kind, job_token_scopes) + VALUES ('test-workspace', 'test-user-3', 'export function main() {}', '{}', '', '', + $1, $2, 'deno', '', 'script', $3)", + ) + .bind(path) + .bind(hash) + .bind(scopes) + .execute(db) + .await?; + Ok(()) +} + +#[sqlx::test(fixtures("base"))] +async fn test_restricted_job_token_is_confined(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + insert_script(&db, "u/test-user-3/open", 515151, None).await?; + insert_script( + &db, + "u/test-user-3/oidc_only", + 525252, + Some(&["oidc:write"]), + ) + .await?; + insert_job(&db, FLOW_JOB, None, &["oidc:write"]).await?; + insert_job(&db, OIDC_JOB, Some(FLOW_JOB), &["oidc:write"]).await?; + insert_job(&db, RUN_JOB, None, &["jobs:run"]).await?; + + let server = ApiServer::start(db.clone()).await?; + set_jwt_secret().await; + let base = format!( + "http://localhost:{}/api/w/test-workspace", + server.addr.port() + ); + let client = reqwest::Client::new(); + let oidc_token = job_token(&db, OIDC_JOB).await?; + + let refused = [ + client.get(format!("{base}/variables/get_value/u/test-user-3/secret")), + client + .post(format!("{base}/scripts/create")) + .json(&json!({})), + client + .post(format!("{base}/schedules/create")) + .json(&json!({})), + client + .post(format!("{base}/jobs/run/p/u/test-user-3/open")) + .json(&json!({})), + ]; + for request in refused { + let resp = request.bearer_auth(&oidc_token).send().await?; + assert_eq!( + resp.status(), + StatusCode::FORBIDDEN, + "{}", + resp.text().await? + ); + } + let own = client + .get(format!("{base}/jobs_u/get_root_job_id/{OIDC_JOB}")) + .bearer_auth(&oidc_token) + .send() + .await?; + assert_eq!(own.status(), StatusCode::OK, "{}", own.text().await?); + // The orchestrator reads a flow's step results with the token of the step that just ran. + for (job, refused) in [(FLOW_JOB, false), (RUN_JOB, true)] { + let resp = client + .get(format!("{base}/jobs/result_by_id/{job}/a")) + .bearer_auth(&oidc_token) + .send() + .await?; + assert_eq!(resp.status() == StatusCode::FORBIDDEN, refused, "{job}"); + } + // Its progress reaches only the flow it runs in. + for (flow, refused) in [(FLOW_JOB, false), (RUN_JOB, true)] { + let resp = client + .post(format!("{base}/job_metrics/set_progress/{OIDC_JOB}")) + .bearer_auth(&oidc_token) + .json(&json!({ "percent": 50, "flow_job_id": flow })) + .send() + .await?; + assert_eq!(resp.status() == StatusCode::FORBIDDEN, refused, "{flow}"); + } + + // Imported queue rows bypass push, so a restricted token cannot import, even an admin's. + let admin_job = Uuid::parse_str("b0000000-0000-0000-0000-000000000004")?; + sqlx::query( + "INSERT INTO v2_job (id, workspace_id, created_by, permissioned_as, permissioned_as_email, + kind, script_lang, runnable_path, tag) + VALUES ($1, 'test-workspace', 'test-user', 'u/test-user', 'test@windmill.dev', + 'script', 'deno', 'u/test-user/agent', 'deno')", + ) + .bind(admin_job) + .execute(&db) + .await?; + sqlx::query( + "INSERT INTO job_perms (job_id, email, username, is_admin, is_operator, folders, groups, + workspace_id, job_token_scopes) + VALUES ($1, 'test@windmill.dev', 'test-user', true, false, '{}', '{}', 'test-workspace', + '{jobs:write,jobs:run}')", + ) + .bind(admin_job) + .execute(&db) + .await?; + let admin_token = windmill_common::auth::create_token_for_owner( + &db, + "test-workspace", + "u/test-user", + "ephemeral-script", + 300, + "test@windmill.dev", + &admin_job, + None, + None, + ) + .await?; + let resp = client + .post(format!("{base}/jobs/queue/import")) + .bearer_auth(&admin_token) + .json(&json!([])) + .send() + .await?; + assert_eq!(resp.status(), StatusCode::FORBIDDEN); + assert!(resp.text().await?.contains("cannot import")); + // Nor can it, even an admin's, place a child in an unrelated run: the flow-run routes + // trust that lineage. + let resp = client + .post(format!( + "{base}/jobs/run/p/u/test-user-3/open?root_job={RUN_JOB}" + )) + .bearer_auth(&admin_token) + .json(&json!({})) + .send() + .await?; + assert_eq!(resp.status(), StatusCode::CREATED, "{}", resp.text().await?); + let child = Uuid::parse_str(&resp.text().await?)?; + let claimed: bool = sqlx::query_scalar( + "SELECT $2 IN (parent_job, root_job, flow_innermost_root_job) IS TRUE FROM v2_job + WHERE id = $1", + ) + .bind(child) + .bind(Uuid::parse_str(RUN_JOB)?) + .fetch_one(&db) + .await?; + assert!(!claimed); + + // A job it starts is capped at its own scopes, intersected with the target's setting. + let run_token = job_token(&db, RUN_JOB).await?; + for (path, expected) in [ + ("open", vec!["jobs:run".to_string()]), + ("oidc_only", vec![]), + ] { + let resp = client + .post(format!("{base}/jobs/run/p/u/test-user-3/{path}")) + .bearer_auth(&run_token) + .json(&json!({})) + .send() + .await?; + assert_eq!(resp.status(), StatusCode::CREATED); + let child = Uuid::parse_str(&resp.text().await?)?; + let scopes: Option> = + sqlx::query_scalar("SELECT job_token_scopes FROM job_perms WHERE job_id = $1") + .bind(child) + .fetch_one(&db) + .await?; + assert_eq!(scopes, Some(expected), "child of {path}"); + } + Ok(()) +} + +#[sqlx::test(fixtures("base"))] +async fn test_flow_steps_inherit_the_flow_restriction(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let value = json!({ "modules": [ + { "id": "a", "value": { "type": "identity" }, + "job_token_scopes": ["oidc:write", "variables:read"] }, + { "id": "b", "value": { "type": "identity" } }, + ] }); + sqlx::query( + "INSERT INTO flow (workspace_id, path, summary, description, value, edited_by, edited_at, + schema, extra_perms, versions) + VALUES ('test-workspace', 'u/test-user/agent_flow', '', '', $1, 'test-user', now(), '{}', + '{}', '{7171}')", + ) + .bind(&value) + .execute(&db) + .await?; + sqlx::query( + "INSERT INTO flow_version (id, workspace_id, path, value, schema, created_by) + VALUES (7171, 'test-workspace', 'u/test-user/agent_flow', $1, '{}', 'test-user')", + ) + .bind(&value) + .execute(&db) + .await?; + + let server = ApiServer::start(db.clone()).await?; + let scopes = |v: &[&str]| Some(v.iter().map(|s| s.to_string()).collect::>()); + // A step's own setting narrows the flow's restriction and never widens it; in an + // unrestricted flow it restricts that step alone. + for (flow_scopes, expected_a, expected_b) in [ + (scopes(&["oidc:write"]), scopes(&["oidc:write"]), scopes(&["oidc:write"])), + (None, scopes(&["oidc:write", "variables:read"]), None), + ] { + let flow = RunJob::from(JobPayload::Flow { + path: "u/test-user/agent_flow".to_string(), + dedicated_worker: None, + apply_preprocessor: false, + version: 7171, + labels: None, + job_token_scopes: flow_scopes.clone(), + }) + .run_until_complete(&db, false, server.addr.port()) + .await; + + let steps: Vec<(String, Option>)> = sqlx::query_as( + "SELECT j.flow_step_id, p.job_token_scopes FROM v2_job j + JOIN job_perms p ON p.job_id = j.id + WHERE j.parent_job = $1 ORDER BY j.flow_step_id", + ) + .bind(flow.id) + .fetch_all(&db) + .await?; + assert_eq!( + steps, + vec![("a".to_string(), expected_a), ("b".to_string(), expected_b)], + "flow scopes {flow_scopes:?}" + ); + } + Ok(()) +} + +#[sqlx::test(fixtures("base"))] +async fn test_deploy_without_the_field_keeps_the_restriction( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let base = format!("http://localhost:{}/api/w/test-workspace", server.addr.port()); + let client = reqwest::Client::new(); + let script = |path: &str, scopes: Option| { + let mut body = json!({ + "path": path, "summary": "", "description": "", "content": "echo 42", "language": "bash", + }); + if let Some(scopes) = scopes { + body["job_token_scopes"] = scopes; + } + body + }; + let stored = |path: &'static str| { + let db = db.clone(); + async move { + sqlx::query_scalar::<_, Option>>( + "SELECT job_token_scopes FROM script WHERE path = $1 ORDER BY created_at DESC LIMIT 1", + ) + .bind(path) + .fetch_one(&db) + .await + } + }; + + let resp = client + .post(format!("{base}/scripts/create")) + .bearer_auth("SECRET_TOKEN") + .json(&script("u/test-user/agent", Some(json!(["oidc:write"])))) + .send() + .await?; + assert_eq!(resp.status(), StatusCode::CREATED, "{}", resp.text().await?); + + // A rename by a client that does not know the field carries the restriction along. + let resp = client + .post(format!("{base}/scripts/update/u/test-user/agent")) + .bearer_auth("SECRET_TOKEN") + .json(&script("u/test-user/renamed", None)) + .send() + .await?; + assert!(resp.status().is_success(), "{}", resp.text().await?); + assert_eq!(stored("u/test-user/renamed").await?, Some(vec!["oidc:write".to_string()])); + + // An explicit null clears it. + let resp = client + .post(format!("{base}/scripts/update/u/test-user/renamed")) + .bearer_auth("SECRET_TOKEN") + .json(&script("u/test-user/renamed", Some(serde_json::Value::Null))) + .send() + .await?; + assert!(resp.status().is_success(), "{}", resp.text().await?); + assert_eq!(stored("u/test-user/renamed").await?, None); + Ok(()) +} + +/// The OIDC `job_token_scopes` claim states the scopes of the token that asked for it. +#[cfg(all(feature = "enterprise", feature = "private", feature = "openidconnect"))] +#[sqlx::test(fixtures("base"))] +async fn test_oidc_claim_carries_the_presented_scopes(db: Pool) -> anyhow::Result<()> { + use base64::Engine; + + initialize_tracing().await; + const OPEN_JOB: &str = "b0000000-0000-0000-0000-000000000005"; + insert_script(&db, "u/test-user-3/agent", 535353, None).await?; + insert_job(&db, OIDC_JOB, None, &["oidc:write"]).await?; + insert_job_with(&db, OPEN_JOB, None, None).await?; + sqlx::query("UPDATE v2_job SET runnable_id = 535353 WHERE id = ANY($1)") + .bind([Uuid::parse_str(OIDC_JOB)?, Uuid::parse_str(OPEN_JOB)?]) + .execute(&db) + .await?; + sqlx::query( + "INSERT INTO v2_job_queue (id, workspace_id, running, scheduled_for, tag) + SELECT id, workspace_id, true, now(), tag FROM v2_job WHERE id = ANY($1)", + ) + .bind([Uuid::parse_str(OIDC_JOB)?, Uuid::parse_str(OPEN_JOB)?]) + .execute(&db) + .await?; + + let server = ApiServer::start(db.clone()).await?; + set_jwt_secret().await; + // The issuer of the minted token. + windmill_common::BASE_URL.store(std::sync::Arc::new("http://localhost".to_string())); + let base = format!( + "http://localhost:{}/api/w/test-workspace", + server.addr.port() + ); + let client = reqwest::Client::new(); + for (job, expected) in [(OIDC_JOB, json!(["oidc:write"])), (OPEN_JOB, json!(null))] { + let resp = client + .post(format!("{base}/oidc/token/test-audience")) + .bearer_auth(job_token(&db, job).await?) + .send() + .await?; + let status = resp.status(); + let id_token = resp.text().await?; + assert_eq!(status, StatusCode::OK, "{id_token}"); + let payload = id_token.split('.').nth(1).expect("a JWT payload"); + let claims: serde_json::Value = serde_json::from_slice( + &base64::engine::general_purpose::URL_SAFE_NO_PAD.decode(payload)?, + )?; + assert_eq!(claims["job_token_scopes"], expected, "{job}"); + } + + // A soft-canceled job is still `running` until its worker kills it. + sqlx::query("UPDATE v2_job_queue SET canceled_by = 'test-user' WHERE id = $1") + .bind(Uuid::parse_str(OIDC_JOB)?) + .execute(&db) + .await?; + let resp = client + .post(format!("{base}/oidc/token/test-audience")) + .bearer_auth(job_token(&db, OIDC_JOB).await?) + .send() + .await?; + assert_eq!( + resp.status(), + StatusCode::FORBIDDEN, + "{}", + resp.text().await? + ); + Ok(()) +} diff --git a/backend/tests/jobs_read_auth.rs b/backend/tests/jobs_read_auth.rs index fd2969a18f..bffacfbaf0 100644 --- a/backend/tests/jobs_read_auth.rs +++ b/backend/tests/jobs_read_auth.rs @@ -476,6 +476,44 @@ async fn test_single_job_read_authorization(db: Pool) -> anyhow::Resul ); } + // A lock job belongs to `jobs:run:dependencies`: its path is whatever the caller sent, + // so a token scoped to run the flow at that path must not read it. + let path = "completed/get_result/19191919-1919-1919-1919-191919191919"; + let (status, body) = get(&base, path, Some("RUN_SCOPED_DEPS_TOKEN")).await; + assert!( + status.is_success() && body.contains("DEPS_RESULT"), + "dependencies-scoped token must read its lock job (got {status}): {body}" + ); + let (status, body) = get(&base, path, Some("RUN_SCOPED_TOKEN")).await; + assert_eq!( + status, + reqwest::StatusCode::NOT_FOUND, + "a flow-scoped token must not read a lock job (got {status}): {body}" + ); + let (status, body) = get( + &base, + &format!("completed/get_result/{FLOW_JOB}"), + Some("RUN_SCOPED_DEPS_TOKEN"), + ) + .await; + assert_eq!( + status, + reqwest::StatusCode::NOT_FOUND, + "dependencies-scoped token must not read a flow run at its path (got {status}): {body}" + ); + // A lock job filed at `*` matches no path-scoped grant: the stored path is not a pattern. + let (status, body) = get( + &base, + "completed/get_result/20202020-2020-2020-2020-202020202020", + Some("RUN_SCOPED_DEPS_TOKEN"), + ) + .await; + assert_eq!( + status, + reqwest::StatusCode::NOT_FOUND, + "a path-scoped token must not read a lock job filed at `*` (got {status}): {body}" + ); + // An `apps:run:` scope is a start grant too: the inline-script component run it // launched — a kind no `jobs:run` scope can name — stays readable to a token scoped // to that app, and stays out of reach for one that is only scoped to run jobs. @@ -564,6 +602,56 @@ async fn test_single_job_read_authorization(db: Pool) -> anyhow::Resul "run-scoped token must not enumerate jobs (got {status}): {body}" ); + // ---- PATH-SCOPED READ TOKEN: READ_SCOPED_TOKEN is test-user-2's + // `jobs:read:f/shared/flow1`. It reads that flow's runs and the steps beneath + // them, and nothing else its owner created. + for (path, expected) in [ + ( + format!("completed/get_result/{FLOW_JOB}"), + r#""flow": "done""#, + ), + ( + format!("completed/get_result/{STEP_JOB}"), + "STEP_RESULT_INHERITED", + ), + ] { + let (status, body) = get(&base, &path, Some("READ_SCOPED_TOKEN")).await; + assert!( + status.is_success() && body.contains(expected), + "read-scoped token must read a run of its flow ({path}, got {status}): {body}" + ); + } + for path in [ + format!("completed/get_result/{VICTIM}"), + format!("get/{VICTIM}"), + format!("flow/approval_info/{VICTIM}"), + ] { + let (status, body) = get(&base, &path, Some("READ_SCOPED_TOKEN")).await; + assert_eq!( + status, + reqwest::StatusCode::NOT_FOUND, + "read-scoped token must not read a job outside its paths ({path}, got {status}): {body}" + ); + } + // The lists answer with the top-level runs at its paths only, where the same user's + // unscoped token sees the other runs too. + for list in ["list", "completed/list"] { + let (status, body) = get(&authed_base, list, Some("READ_SCOPED_TOKEN")).await; + assert!(status.is_success(), "{list} (got {status}): {body}"); + let jobs: Vec = serde_json::from_str(&body)?; + let mut ids: Vec<&str> = jobs.iter().filter_map(|j| j["id"].as_str()).collect(); + ids.sort(); + assert_eq!(ids, [WRAPPED_FLOW_JOB, FLOW_JOB], "{list}: {body}"); + } + let (_, body) = get(&authed_base, "list", Some("SECRET_TOKEN_2")).await; + assert!(body.contains(VICTIM), "unscoped list: {body}"); + let (status, body) = get(&authed_base, "completed/count", Some("READ_SCOPED_TOKEN")).await; + assert_eq!( + status, + reqwest::StatusCode::FORBIDDEN, + "read-scoped token must not count the workspace's jobs (got {status}): {body}" + ); + // ---- APP EMBED TOKEN: cancellation confined to the app's own jobs. The token // may cancel a job it launched (created_by == viewer), but `cancel_job_api` // denies (NotFound) a job created by someone else, even one the (admin) diff --git a/backend/tests/list_jobs.rs b/backend/tests/list_jobs.rs index 29a49c1610..d33618d048 100644 --- a/backend/tests/list_jobs.rs +++ b/backend/tests/list_jobs.rs @@ -957,6 +957,7 @@ async fn test_job_labels_propagated_at_push_time(db: Pool) -> anyhow:: .into(), debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), labels: Some(vec!["prod".to_string(), "deploy".to_string()]), + job_token_scopes: None, }) .push(&db) .await; @@ -1060,6 +1061,7 @@ async fn test_job_label_filter(db: Pool) -> anyhow::Result<()> { .into(), debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), labels: Some(vec!["prod".to_string()]), + job_token_scopes: None, }) .push(&db) .await; @@ -1077,6 +1079,7 @@ async fn test_job_label_filter(db: Pool) -> anyhow::Result<()> { .into(), debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), labels: Some(vec!["staging".to_string()]), + job_token_scopes: None, }) .push(&db) .await; diff --git a/backend/tests/mcp_header_var_refs.rs b/backend/tests/mcp_header_var_refs.rs new file mode 100644 index 0000000000..158bc71a5f --- /dev/null +++ b/backend/tests/mcp_header_var_refs.rs @@ -0,0 +1,110 @@ +//! An MCP resource header whose value is a `$var:` reference reaches the MCP +//! server as the variable's value, and only for a caller allowed to read it. +#![cfg(feature = "mcp")] + +use sqlx::{Pool, Postgres}; +use tokio::io::AsyncReadExt; +use windmill_common::client::AuthedClient; +use windmill_test_utils::*; +use windmill_worker::{load_mcp_tools, McpResourceConfig}; + +const SECRET_VALUE: &str = "S3CRET-MCP-HEADER-VALUE"; + +/// A secret variable and an MCP resource at `url` sending it as a header. +async fn seed(db: &Pool, url: &str) -> anyhow::Result<()> { + let mc = windmill_common::variables::build_crypt(db, "test-workspace").await?; + sqlx::query( + "INSERT INTO variable (workspace_id, path, value, is_secret, description, extra_perms) + VALUES ('test-workspace', 'u/test-user/mcp_header', $1, true, '', '{}')", + ) + .bind(windmill_common::variables::encrypt(&mc, SECRET_VALUE)) + .execute(db) + .await?; + sqlx::query( + "INSERT INTO resource (workspace_id, path, value, resource_type, extra_perms, created_by) + VALUES ('test-workspace', 'u/test-user/mcp', $1, 'mcp', '{}', 'test-user')", + ) + .bind(serde_json::json!({ + "name": "mcp", + "url": url, + "headers": { "X-Api-Key": "$var:u/test-user/mcp_header", "X-Plain": "plain" } + })) + .execute(db) + .await?; + Ok(()) +} + +#[sqlx::test(fixtures("base"))] +async fn test_mcp_header_var_ref_is_resolved(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + std::env::set_var("ALLOW_PRIVATE_MCP_SERVER_URLS", "true"); + + // Not an MCP server: it captures the first request and hangs up, so loading + // fails at the connect step after the headers were sent. + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await?; + let mcp_port = listener.local_addr()?.port(); + let first_request = tokio::spawn(async move { + let (mut socket, _) = listener.accept().await?; + let mut buf = vec![0u8; 16 * 1024]; + let n = socket.read(&mut buf).await?; + anyhow::Ok(String::from_utf8_lossy(&buf[..n]).to_lowercase()) + }); + seed(&db, &format!("http://127.0.0.1:{mcp_port}/mcp")).await?; + + let server = ApiServer::start(db.clone()).await?; + let client = AuthedClient::new( + format!("http://localhost:{}", server.addr.port()), + "test-workspace".to_string(), + "SECRET_TOKEN".to_string(), + None, + ); + let config = vec![McpResourceConfig { + resource_path: "$res:u/test-user/mcp".to_string(), + include_tools: None, + exclude_tools: None, + }]; + let _ = load_mcp_tools(&db, "test-workspace", config, &client).await; + + let request = first_request.await??; + assert!( + request.contains(&format!("x-api-key: {}", SECRET_VALUE.to_lowercase())), + "the header must carry the variable's value, not the reference: {request}" + ); + assert!(request.contains("x-plain: plain"), "{request}"); + + Ok(()) +} + +/// A token scoped to the resource alone must not have the API read the +/// variable a header points at. +#[sqlx::test(fixtures("base"))] +async fn test_mcp_header_var_ref_requires_variable_scope(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + seed(&db, "https://mcp.invalid.windmill.test").await?; + sqlx::query( + "INSERT INTO token (token_hash, token_prefix, token, email, label, super_admin, scopes) + VALUES (encode(sha256('RES_ONLY_TOKEN'::bytea), 'hex'), 'RES_ONLY_T', 'RES_ONLY_TOKEN', + 'test@windmill.dev', 'resources only', false, '{resources:read}')", + ) + .execute(&db) + .await?; + + let server = ApiServer::start(db.clone()).await?; + let resp = reqwest::Client::new() + .get(format!( + "http://localhost:{}/api/w/test-workspace/resources/mcp_tools/u/test-user/mcp", + server.addr.port() + )) + .header("Authorization", "Bearer RES_ONLY_TOKEN") + .send() + .await?; + let status = resp.status(); + let body = resp.text().await?; + assert_eq!(status, reqwest::StatusCode::FORBIDDEN, "{body}"); + assert!( + body.contains("variables:read:u/test-user/mcp_header"), + "{body}" + ); + + Ok(()) +} diff --git a/backend/tests/nativets_stress.rs b/backend/tests/nativets_stress.rs index 21d3911778..d1e371ef81 100644 --- a/backend/tests/nativets_stress.rs +++ b/backend/tests/nativets_stress.rs @@ -191,6 +191,7 @@ async fn push_job(db: &Pool, content: &str, args: &serde_json::Value) None, None, None, + None, ) .await .expect("push must succeed"); diff --git a/backend/tests/otel.rs b/backend/tests/otel.rs index 1a4040d390..30585c08a1 100644 --- a/backend/tests/otel.rs +++ b/backend/tests/otel.rs @@ -168,6 +168,7 @@ async fn test_all_metrics_e2e() { // ── Gauges ────────────────────────────────────────────────────── otel_set_queue_count("python3", 42); + otel_set_queue_delay("python3", 12.5); otel_set_queue_running_count("deno", 5); otel_set_worker_busy("worker-test-1", 1); otel_set_db_pool(5, 10, 20); @@ -175,6 +176,10 @@ async fn test_all_metrics_e2e() { otel_set_worker_uptime("w-uptime", 3600.0); otel_set_health_status_phase("healthy"); otel_set_health_db_unresponsive(true); + otel_set_worker_memory("w-res", "gpu", Some(512), Some(128), Some(1024)); + otel_add_worker_cpu("w-res", "gpu", 1.5, 40, 7, 0.35); + otel_set_worker_occupancy_rate("w-res", "gpu", "5m", 0.25); + otel_record_job_memory_peak("python3", 100 * 1024 * 1024); // ── Histograms ────────────────────────────────────────────────── @@ -188,7 +193,7 @@ async fn test_all_metrics_e2e() { let metrics = flush_and_get_metrics(&state); let names = metric_names(&metrics); - // ── Verify all 20 metric names are present ────────────────────── + // ── Verify all metric names are present ───────────────────────── let expected = [ "windmill.queue.push_count", @@ -197,6 +202,7 @@ async fn test_all_metrics_e2e() { "windmill.queue.zombie_restart_count", "windmill.queue.zombie_delete_count", "windmill.queue.count", + "windmill.queue.delay", "windmill.queue.running_count", "windmill.worker.execution_count", "windmill.worker.execution_duration", @@ -211,6 +217,15 @@ async fn test_all_metrics_e2e() { "windmill.worker.uptime", "windmill.health.status", "windmill.health.db_unresponsive", + "windmill.worker.memory_usage", + "windmill.worker.windmill_memory_usage", + "windmill.worker.memory_limit", + "windmill.worker.cpu_usage", + "windmill.worker.cpu_periods", + "windmill.worker.cpu_throttled_periods", + "windmill.worker.cpu_throttled_time", + "windmill.worker.occupancy_rate", + "windmill.worker.job_memory_peak", ]; for name in expected { assert!( @@ -261,6 +276,9 @@ async fn test_all_metrics_e2e() { .expect("queue.count data point with tag=python3 not found"); assert_eq!(dp.1, 42); + let m = find_metric(&metrics, "windmill.queue.delay").unwrap(); + assert!((gauge_f64_value(m).unwrap() - 12.5).abs() < f64::EPSILON); + let m = find_metric(&metrics, "windmill.queue.running_count").unwrap(); let values = gauge_i64_values(m); let dp = values @@ -285,6 +303,18 @@ async fn test_all_metrics_e2e() { .expect("worker.busy data point with worker=worker-test-1 not found"); assert_eq!(dp.1, 1); + let m = find_metric(&metrics, "windmill.worker.memory_limit").unwrap(); + let values = gauge_i64_values(m); + let dp = values + .iter() + .find(|(attrs, _)| { + attrs + .iter() + .any(|kv| kv.key.as_str() == "worker_group" && kv.value.as_str() == "gpu") + }) + .expect("worker.memory_limit data point with worker_group=gpu not found"); + assert_eq!(dp.1, 1024); + let m = find_metric(&metrics, "windmill.db.pool.active").unwrap(); assert_eq!(gauge_i64_values(m)[0].1, 5); let m = find_metric(&metrics, "windmill.db.pool.idle").unwrap(); diff --git a/backend/tests/preview_native_tag.rs b/backend/tests/preview_native_tag.rs index eba686396a..39f985cefe 100644 --- a/backend/tests/preview_native_tag.rs +++ b/backend/tests/preview_native_tag.rs @@ -76,6 +76,7 @@ async fn push_preview_and_get_row( None, None, None, + None, ) .await .expect("push must succeed"); diff --git a/backend/tests/python_jobs.rs b/backend/tests/python_jobs.rs index 9616df267a..4236599aba 100644 --- a/backend/tests/python_jobs.rs +++ b/backend/tests/python_jobs.rs @@ -496,6 +496,28 @@ async def main(x: int): ); } + // `hasattr` does not swallow the KeyError this `__getattr__` raises; released + // `wmill` clients ship an `S3Object` built this way. + #[test] + fn test_python_execd_result_with_keyerror_getattr() { + let script = r#" +class AttrDict(dict): + def __getattr__(self, attr): + return self[attr] + +def main(x: str): + return AttrDict(s3=x) +"#; + let results = run_py_raw_protocol_test( + &[("f/test/attr_dict", script)], + vec![ProtocolCmd::Execd { args: serde_json::json!({"x": "a/b.txt"}) }], + ); + assert_eq!( + results, + vec![DedicatedWorkerResult::Success(serde_json::json!({"s3": "a/b.txt"}))] + ); + } + // ==================== Argument Transformation Tests ==================== #[test] @@ -1278,6 +1300,7 @@ async fn test_python_wac_v2_with_preprocessor(db: Pool) -> anyhow::Res debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), labels: None, + job_token_scopes: None, }) .arg("who", json!("alice")) .arg("count", json!(7)) diff --git a/backend/tests/restart_at_step_api.rs b/backend/tests/restart_at_step_api.rs index c5ab703f46..f6fc2a04d3 100644 --- a/backend/tests/restart_at_step_api.rs +++ b/backend/tests/restart_at_step_api.rs @@ -41,6 +41,7 @@ async fn test_api_restart_at_step_nested_happy(db: Pool) -> anyhow::Re apply_preprocessor: true, version: 1443253234253454, labels: None, + job_token_scopes: None, }) .run_until_complete(&db, false, port) .await; @@ -85,6 +86,7 @@ async fn test_api_restart_at_step_top_level_happy(db: Pool) -> anyhow: apply_preprocessor: true, version: 1443253234253454, labels: None, + job_token_scopes: None, }) .run_until_complete(&db, false, port) .await; @@ -118,6 +120,7 @@ async fn test_api_restart_at_step_rejects_unknown_step(db: Pool) -> an apply_preprocessor: true, version: 1443253234253454, labels: None, + job_token_scopes: None, }) .run_until_complete(&db, false, port) .await; @@ -161,6 +164,7 @@ async fn test_api_restart_at_step_rejects_out_of_range_iteration( apply_preprocessor: true, version: 1443253234253454, labels: None, + job_token_scopes: None, }) .run_until_complete(&db, false, port) .await; @@ -213,7 +217,7 @@ async fn test_api_restart_at_step_rejects_parallel_loop(db: Pool) -> a "id": "inner", "value": { "type": "rawscript", - "language": "deno", + "language": "bun", "input_transforms": {}, "content": "export function main() { return 'ok' }" } diff --git a/backend/tests/restart_flow_version_scope.rs b/backend/tests/restart_flow_version_scope.rs index 82e8f8e4e7..551e8c3619 100644 --- a/backend/tests/restart_flow_version_scope.rs +++ b/backend/tests/restart_flow_version_scope.rs @@ -53,6 +53,7 @@ async fn push_restart( None, None, None, + None, ) .await?; tx.commit().await?; diff --git a/backend/tests/retry.rs b/backend/tests/retry.rs index e804b96f0e..d80e386b61 100644 --- a/backend/tests/retry.rs +++ b/backend/tests/retry.rs @@ -70,12 +70,20 @@ mod retry { fn inner_step() -> &'static str { r#" +import { connect } from "node:net"; + export async function main(index, port) { - const buf = new Uint8Array([0]); - const sock = await Deno.connect({ port }); - await sock.write(new Uint8Array([index])); - if (await sock.read(buf) != 1) throw Error("read"); - return buf[0]; + return await new Promise((resolve, reject) => { + const sock = connect({ port, host: "127.0.0.1" }, () => { + sock.write(new Uint8Array([index])); + }); + sock.on("data", (data) => { + resolve(data[0]); + sock.destroy(); + }); + sock.on("error", reject); + sock.on("close", () => reject(Error("read"))); + }); } "# } @@ -105,7 +113,7 @@ def main(last, port): "port": { "type": "javascript", "expr": "flow_input.port" }, }, "type": "rawscript", - "language": "deno", + "language": "bun", "content": inner_step(), }, }], @@ -281,7 +289,7 @@ def main(last, port): "value": { "input_transforms": {}, "type": "rawscript", - "language": "deno", + "language": "bun", "content": "export function main() { return \"ok\" }", }, }, { @@ -293,7 +301,7 @@ def main(last, port): "port": { "type": "javascript", "expr": "flow_input.port" }, }, "type": "rawscript", - "language": "deno", + "language": "bun", "content": inner_step(), }, "retry": { "constant": { "attempts": 1, "seconds": 0 } }, diff --git a/backend/tests/success_handler.rs b/backend/tests/success_handler.rs index fe6127c27e..14df1a4c25 100644 --- a/backend/tests/success_handler.rs +++ b/backend/tests/success_handler.rs @@ -2,6 +2,36 @@ use sqlx::{Pool, Postgres}; use windmill_test_utils::*; +/// Both job-running tests seed `f/test/success_handler` with a different hash, each in its own +/// database, while the deployed-script caches are process-global and keyed by workspace and +/// path. With them on, one test resolves the handler to the other's hash and the handler push +/// 404s whenever that hash's info is not cached yet. +#[cfg(all(feature = "enterprise", feature = "private"))] +fn disable_deployed_script_cache() { + windmill_common::DEPLOYED_SCRIPT_CACHE_DISABLED + .store(true, std::sync::atomic::Ordering::Relaxed); +} + +/// The handler job is pushed after the job it handles is reported complete. +#[cfg(all(feature = "enterprise", feature = "private"))] +async fn wait_for_success_handler_job( + db: &Pool, +) -> anyhow::Result, Option)>> { + for _ in 0..100 { + let job = sqlx::query_as::<_, (Option, Option)>( + "SELECT runnable_path, parent_job FROM v2_job + WHERE workspace_id = 'test-workspace' AND permissioned_as_email = 'success_handler@windmill.dev'", + ) + .fetch_optional(db) + .await?; + if job.is_some() { + return Ok(job); + } + tokio::time::sleep(tokio::time::Duration::from_millis(100)).await; + } + Ok(None) +} + /// Test that the workspace success handler cache works correctly with 60s TTL #[cfg(feature = "deno_core")] #[sqlx::test(fixtures("base"))] @@ -108,6 +138,7 @@ async fn test_success_handler_triggered_on_success(db: Pool) -> anyhow use windmill_common::scripts::{ScriptHash, ScriptLang}; initialize_tracing().await; + disable_deployed_script_cache(); let server = ApiServer::start(db.clone()).await?; @@ -123,7 +154,7 @@ export async function main(path: string, email: string, job_id: string, is_flow: sqlx::query!( r#" INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by, schema, summary, description, lock) - VALUES ('test-workspace', 1234567890, 'f/test/success_handler', $1, 'deno', 'script', 'test-user', '{}', 'Success handler script', 'Handles successful job completions', '') + VALUES ('test-workspace', 1234567890, 'f/test/success_handler', $1, 'bun', 'script', 'test-user', '{}', 'Success handler script', 'Handles successful job completions', E'{}\n//bun.lock\n') "#, success_handler_code ) @@ -138,7 +169,7 @@ export async function main(path: string, email: string, job_id: string, is_flow: sqlx::query!( r#" INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by, schema, summary, description, lock) - VALUES ('test-workspace', $1, 'f/test/simple_script', $2, 'deno', 'script', 'test-user', '{}', 'Simple test script', 'A simple test script', '') + VALUES ('test-workspace', $1, 'f/test/simple_script', $2, 'bun', 'script', 'test-user', '{}', 'Simple test script', 'A simple test script', E'{}\n//bun.lock\n') "#, test_script_hash, test_script_code @@ -175,12 +206,13 @@ export async function main(path: string, email: string, job_id: string, is_flow: cache_ttl: None, cache_ignore_s3_path: None, dedicated_worker: None, - language: ScriptLang::Deno, + language: ScriptLang::Bun, priority: None, apply_preprocessor: false, concurrency_settings: ConcurrencySettings::default(), debouncing_settings: DebouncingSettings::default(), labels: None, + job_token_scopes: None, }) .run_until_complete(&db, false, server.addr.port()) .await; @@ -190,51 +222,72 @@ export async function main(path: string, email: string, job_id: string, is_flow: let main_job_id = completed_job.id; - // Wait a short time for the success handler job to be created - tokio::time::sleep(tokio::time::Duration::from_millis(500)).await; - - // Verify the success handler job was created (query by email since trigger is not set) - let success_handler_job = sqlx::query!( - r#" - SELECT - id, - runnable_path, - permissioned_as_email, - parent_job - FROM v2_job - WHERE workspace_id = 'test-workspace' - AND permissioned_as_email = 'success_handler@windmill.dev' - ORDER BY created_at DESC - LIMIT 1 - "# - ) - .fetch_optional(&db) - .await?; - - assert!( - success_handler_job.is_some(), - "Success handler job should have been created" - ); - - let handler_job = success_handler_job.unwrap(); - - // Verify the success handler job has correct parameters + let handler_job = wait_for_success_handler_job(&db).await?; assert_eq!( - handler_job.runnable_path.as_deref(), - Some("f/test/success_handler"), - "Success handler should run the configured script" - ); - assert_eq!( - handler_job.permissioned_as_email.as_str(), - "success_handler@windmill.dev", - "Success handler should run as success_handler user" - ); - assert_eq!( - handler_job.parent_job, - Some(main_job_id), - "Success handler should have main job as parent" + handler_job, + Some(( + Some("f/test/success_handler".to_string()), + Some(main_job_id) + )), + "Success handler should run the configured script with the main job as parent" ); // Note: root_job may be None when it equals parent_job (optimization in push function) Ok(()) } + +/// A hub script run at the top level (a schedule, a trigger, the API) is a `script_hub` job, +/// and its success must reach the workspace success handler like a workspace script's. +#[cfg(all(feature = "enterprise", feature = "private"))] +#[sqlx::test(fixtures("base"))] +async fn test_success_handler_triggered_on_hub_script_success( + db: Pool, +) -> anyhow::Result<()> { + use windmill_common::jobs::JobPayload; + + initialize_tracing().await; + disable_deployed_script_cache(); + + let server = ApiServer::start(db.clone()).await?; + + let version = "990000003"; + let hub_dir = &*windmill_common::worker::HUB_CACHE_DIR; + tokio::fs::create_dir_all(hub_dir).await?; + tokio::fs::write( + format!("{hub_dir}/{version}"), + r#"{"content":"echo ok","lockfile":null,"language":"bash","schema":{},"summary":null}"#, + ) + .await?; + + // The workspace handler settings are cached per workspace id across the tests of this + // binary, so this reuses the handler path every other test here configures. + sqlx::query( + "INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by, schema, summary, description, lock) + VALUES ('test-workspace', 1234567891, 'f/test/success_handler', 'echo handled', 'bash', 'script', 'test-user', '{}', '', '', '')", + ) + .execute(&db) + .await?; + sqlx::query( + r#"UPDATE workspace_settings SET success_handler = '{"path": "script/f/test/success_handler"}'::jsonb + WHERE workspace_id = 'test-workspace'"#, + ) + .execute(&db) + .await?; + + let completed_job = RunJob::from(JobPayload::ScriptHub { + path: format!("hub/{version}/test/ok"), + apply_preprocessor: false, + }) + .run_until_complete(&db, false, server.addr.port()) + .await; + assert!(completed_job.success, "Hub script should have succeeded"); + + let handler_job = wait_for_success_handler_job(&db).await?; + assert_eq!( + handler_job.map(|(_, parent)| parent), + Some(Some(completed_job.id)), + "Workspace success handler should handle the successful hub script" + ); + + Ok(()) +} diff --git a/backend/tests/suspend_resume.rs b/backend/tests/suspend_resume.rs index 6fb5d68004..0decc16519 100644 --- a/backend/tests/suspend_resume.rs +++ b/backend/tests/suspend_resume.rs @@ -63,11 +63,11 @@ mod suspend_resume { "op": { "type": "javascript", "expr": "flow_input.op ?? 'resume'", }, }, "type": "rawscript", - "language": "deno", + "language": "bun", "content": "\ export async function main(n, port, op) {\ - const job = Deno.env.get('WM_JOB_ID'); - const token = Deno.env.get('WM_TOKEN'); + const job = process.env.WM_JOB_ID; + const token = process.env.WM_TOKEN; const r = await fetch( `http://localhost:${port}/api/w/test-workspace/jobs/job_signature/${job}/0?token=${token}&approver=ruben`,\ {\ @@ -102,7 +102,7 @@ mod suspend_resume { "resumes": { "type": "javascript", "expr": "resumes", }, }, "type": "rawscript", - "language": "deno", + "language": "bun", "content": "export function main(n, resume, resumes) { return { n: n + 1, resume, resumes } }" }, "suspend": { @@ -116,7 +116,7 @@ mod suspend_resume { "resumes": { "type": "javascript", "expr": "resumes", }, }, "type": "rawscript", - "language": "deno", + "language": "bun", "content": "export function main(last, resume, resumes) { return { last, resume, resumes } }" }, }], @@ -256,11 +256,11 @@ mod suspend_resume { "port": { "type": "javascript", "expr": "flow_input.port" }, }, "type": "rawscript", - "language": "deno", + "language": "bun", "content": "\ export async function main(port) {\ - const job = Deno.env.get('WM_JOB_ID');\ - const token = Deno.env.get('WM_TOKEN');\ + const job = process.env.WM_JOB_ID;\ + const token = process.env.WM_TOKEN;\ const secret = await (await fetch(\ `http://localhost:${port}/api/w/test-workspace/jobs/job_signature/${job}/0?token=${token}&approver=ruben`,\ { headers: { 'Authorization': `Bearer ${token}` } }\ @@ -278,7 +278,7 @@ mod suspend_resume { "value": { "input_transforms": {}, "type": "rawscript", - "language": "deno", + "language": "bun", "content": "export function main() { return 'b ran' }", }, // The gate holds `b` back, so `b` never runs and its error policy describes @@ -290,7 +290,7 @@ mod suspend_resume { "value": { "input_transforms": {}, "type": "rawscript", - "language": "deno", + "language": "bun", "content": "export function main() { return 'handled' }", }, }, @@ -343,7 +343,7 @@ mod suspend_resume { "id": "a", "value": { "type": "rawscript", - "language": "deno", + "language": "bun", "content": "export function main() { return 'step1'; }" }, "suspend": { @@ -355,7 +355,7 @@ mod suspend_resume { "id": "b", "value": { "type": "rawscript", - "language": "deno", + "language": "bun", "content": "export function main() { return 'step2 - after approval'; }" } }] @@ -454,7 +454,7 @@ mod suspend_resume { "id": "a", "value": { "type": "rawscript", - "language": "deno", + "language": "bun", "content": "export function main() { return 'step1'; }" }, "suspend": { @@ -466,7 +466,7 @@ mod suspend_resume { "id": "b", "value": { "type": "rawscript", - "language": "deno", + "language": "bun", "content": "export function main() { return 'step2 - after approval'; }" } }] @@ -556,7 +556,7 @@ mod suspend_resume { "id": "a", "value": { "type": "rawscript", - "language": "deno", + "language": "bun", "content": "export function main() { return 'step1'; }" }, "suspend": { @@ -567,7 +567,7 @@ mod suspend_resume { "id": "b", "value": { "type": "rawscript", - "language": "deno", + "language": "bun", "content": "export function main() { return 'step2 - after approval'; }" } }] @@ -658,7 +658,7 @@ mod suspend_resume { "id": "a", "value": { "type": "rawscript", - "language": "deno", + "language": "bun", "content": "export function main() { return 'step1'; }" }, "suspend": { @@ -670,7 +670,7 @@ mod suspend_resume { "id": "b", "value": { "type": "rawscript", - "language": "deno", + "language": "bun", "content": "export function main() { return 'step2 - after approval'; }" } }] @@ -764,7 +764,7 @@ mod suspend_resume { "id": "a", "value": { "type": "rawscript", - "language": "deno", + "language": "bun", "content": "export function main() { return 'step1'; }" }, "suspend": { @@ -776,7 +776,7 @@ mod suspend_resume { "id": "b", "value": { "type": "rawscript", - "language": "deno", + "language": "bun", "content": "export function main() { return 'step2 - after approval'; }" } }] @@ -933,7 +933,7 @@ mod suspend_resume { "suspend": { "required_events": 1, "timeout": 86400 }, "value": { "type": "rawscript", - "language": "deno", + "language": "bun", "content": "export async function main() { return 1 }", "input_transforms": {}, }, @@ -942,7 +942,7 @@ mod suspend_resume { "id": "b", "value": { "type": "rawscript", - "language": "deno", + "language": "bun", "content": "export async function main() { return 42 }", "input_transforms": {}, }, @@ -980,7 +980,7 @@ mod suspend_resume { "suspend": { "required_events": APPROVALS, "timeout": 86400 }, "value": { "type": "rawscript", - "language": "deno", + "language": "bun", "content": "export async function main() { return 1 }", "input_transforms": {}, }, @@ -989,7 +989,7 @@ mod suspend_resume { "id": "b", "value": { "type": "rawscript", - "language": "deno", + "language": "bun", "content": "export function main(resumes) { return resumes.length }", "input_transforms": { "resumes": { "type": "javascript", "expr": "resumes" }, diff --git a/backend/tests/v2_job_delete_orphans.rs b/backend/tests/v2_job_delete_orphans.rs index 2b11deab77..45029f024d 100644 --- a/backend/tests/v2_job_delete_orphans.rs +++ b/backend/tests/v2_job_delete_orphans.rs @@ -205,6 +205,75 @@ async fn test_delete_jobs_removes_a_conversation_once_its_last_message_goes( Ok(()) } +/// The agent step completes, and so expires, before the flow run that holds the question: its +/// job going must leave the answer in place, and the run going must take the whole turn. +#[sqlx::test(fixtures("base"))] +async fn test_delete_jobs_removes_a_turn_with_its_flow_run( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + + let flow_job = Uuid::new_v4(); + let step_job = Uuid::new_v4(); + insert_job(&db, WS, flow_job).await?; + insert_job(&db, WS, step_job).await?; + + let conv_id = Uuid::new_v4(); + sqlx::query( + "INSERT INTO flow_conversation (id, workspace_id, flow_path, created_by) + VALUES ($1, $2, 'f/flow', 'test-user')", + ) + .bind(conv_id) + .bind(WS) + .execute(&db) + .await?; + let mut tx = db.begin().await?; + for (job_id, message_type) in [ + ( + flow_job, + windmill_common::flow_conversations::MessageType::User, + ), + ( + step_job, + windmill_common::flow_conversations::MessageType::Assistant, + ), + ] { + windmill_common::flow_conversations::add_message_to_conversation_tx( + &mut tx, + conv_id, + Some(job_id), + flow_job, + "hi", + message_type, + None, + true, + None, + ) + .await?; + } + tx.commit().await?; + + let messages = "SELECT count(*) FROM flow_conversation_message WHERE conversation_id = $1"; + let mut conn = db.acquire().await?; + windmill_common::jobs::delete_jobs(&mut conn, &[step_job]).await?; + drop(conn); + assert_eq!( + count(&db, messages, conv_id).await?, + 2, + "the step's job going must keep the turn" + ); + + let mut conn = db.acquire().await?; + windmill_common::jobs::delete_jobs(&mut conn, &[flow_job]).await?; + drop(conn); + assert_eq!( + count(&db, messages, conv_id).await?, + 0, + "the run going must take the turn" + ); + Ok(()) +} + /// Turns that start while retention is collecting their conversation must land, not fail: /// the conversation lookup locks the row, so each turn waits for the collector's commit, /// finds the conversation gone, and creates it again — the first insert wins and the other @@ -265,6 +334,7 @@ async fn test_new_turns_wait_for_conversation_cleanup_and_recreate( &mut tx, conv_id, Some(new_job), + new_job, "hi again", windmill_common::flow_conversations::MessageType::User, None, diff --git a/backend/tests/wac_suspend_started_at.rs b/backend/tests/wac_suspend_started_at.rs index 5a83730d24..ca364e5993 100644 --- a/backend/tests/wac_suspend_started_at.rs +++ b/backend/tests/wac_suspend_started_at.rs @@ -109,3 +109,91 @@ async fn wac_suspend_stands_down_for_a_cancel(db: Pool) -> anyhow::Res Ok(()) } + +/// Each park clears `started_at`, so the completion would otherwise store only the last +/// segment: a workflow that waited an hour on its tasks would read as a sub-second job that +/// started an hour late. The completed row spans the workflow from its first start, and +/// keeps the last segment aside: that is what is metered and what worker-time readers sum. +#[sqlx::test] +async fn wac_completion_spans_the_workflow_and_keeps_the_last_segment( + db: Pool, +) -> anyhow::Result<()> { + let job_id = Uuid::new_v4(); + sqlx::query( + "INSERT INTO v2_job (id, workspace_id, created_by, created_at, permissioned_as, \ + permissioned_as_email, kind, script_lang, runnable_path, tag, visible_to_owner) \ + VALUES ($1, 'test-workspace', 'test-user', now() - interval '2 hours', 'u/test-user', \ + 'test@windmill.dev', 'script', 'bun', 'u/test-user/wac', 'bun', true)", + ) + .bind(job_id) + .execute(&db) + .await?; + sqlx::query( + "INSERT INTO v2_job_queue (id, workspace_id, scheduled_for, running, tag, started_at) \ + VALUES ($1, 'test-workspace', now(), true, 'bun', now() - interval '1 hour')", + ) + .bind(job_id) + .execute(&db) + .await?; + + // Two parks, each followed by the pull re-stamping `started_at`: the second park must + // not replace the first start with its own segment's. + for resumed_ago in ["10 minutes", "3 seconds"] { + let mut tx = db.begin().await?; + suspend_wac_parent(&mut tx, &job_id, "test-workspace", 1, 3600.0).await?; + tx.commit().await?; + sqlx::query( + "UPDATE v2_job_queue SET suspend = 0, started_at = now() - $2::text::interval \ + WHERE id = $1", + ) + .bind(job_id) + .bind(resumed_ago) + .execute(&db) + .await?; + } + + let job = windmill_queue::get_mini_completed_job(&job_id, "test-workspace", &db) + .await? + .unwrap(); + let result = serde_json::value::to_raw_value(&serde_json::json!("done"))?; + let (_, span) = windmill_queue::add_completed_job( + &db, + &job, + true, + false, + sqlx::types::Json(&result), + None, + 0, + None, + false, + Some(3000), + false, + ) + .await?; + + let (started_ago_s, duration_ms, last_segment_ms): (f64, i64, Option) = sqlx::query_as( + "SELECT extract(epoch FROM now() - started_at)::float8, duration_ms, \ + (extras->>'wac_last_segment_ms')::bigint \ + FROM v2_job_completed WHERE id = $1", + ) + .bind(job_id) + .fetch_one(&db) + .await?; + assert!( + (started_ago_s - 3600.0).abs() < 60.0, + "started_at must be the first start, got {started_ago_s}s ago" + ); + assert!( + (duration_ms - 3_600_000).abs() < 60_000, + "duration_ms must be the wall time since the first start, got {duration_ms}" + ); + assert_eq!(last_segment_ms, Some(3000)); + assert_eq!( + span.duration_ms, duration_ms, + "a flow records the step with the span the row holds" + ); + assert!(span.outlasts_run(job.started_at)); + assert_eq!(span.last_run_ms, 3000, "only the last segment is left to meter"); + + Ok(()) +} diff --git a/backend/tests/wm_token_confinement.rs b/backend/tests/wm_token_confinement.rs index 23d88301b7..e75170ec6f 100644 --- a/backend/tests/wm_token_confinement.rs +++ b/backend/tests/wm_token_confinement.rs @@ -319,7 +319,7 @@ async fn test_wm_token_is_confined_to_its_workspace(db: Pool) -> anyho ); } // ...and the `settings/global` keys on the allowlist, which the CLI reads from a job: on a - // git-sync push, and in `u/admin/hub_sync`. `ws_base_url` is the control: the handler + // git-sync push, and in `wmill hub pull`. `ws_base_url` is the control: the handler // leaves it as ungated as these, so only the allowlist stops it. for key in ["automate_username_creation", "uid", "hub_base_url"] { let resp = authed( diff --git a/backend/tests/worker.rs b/backend/tests/worker.rs index e00ce1172b..751752eb00 100644 --- a/backend/tests/worker.rs +++ b/backend/tests/worker.rs @@ -166,7 +166,7 @@ async fn test_iteration_parallel(db: Pool) -> anyhow::Result<()> { #[cfg(feature = "deno_core")] #[sqlx::test(fixtures("base"))] -async fn test_deno_flow(db: Pool) -> anyhow::Result<()> { +async fn test_bun_flow(db: Pool) -> anyhow::Result<()> { initialize_tracing().await; let server = ApiServer::start(db.clone()).await?; @@ -183,7 +183,7 @@ async fn test_deno_flow(db: Pool) -> anyhow::Result<()> { id: "a".to_string(), value: FlowModuleValue::RawScript { input_transforms: Default::default(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: numbers.to_string(), path: None, lock: None, @@ -212,6 +212,7 @@ async fn test_deno_flow(db: Pool) -> anyhow::Result<()> { apply_preprocessor: None, pass_flow_input_directly: None, debouncing: None, + job_token_scopes: None, }, FlowModule { id: "b".to_string(), @@ -231,7 +232,7 @@ async fn test_deno_flow(db: Pool) -> anyhow::Result<()> { }, )] .into(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: doubles.to_string(), path: None, lock: None, @@ -260,6 +261,7 @@ async fn test_deno_flow(db: Pool) -> anyhow::Result<()> { apply_preprocessor: None, pass_flow_input_directly: None, debouncing: None, + job_token_scopes: None, }], modules_node: None, } @@ -282,6 +284,7 @@ async fn test_deno_flow(db: Pool) -> anyhow::Result<()> { apply_preprocessor: None, pass_flow_input_directly: None, debouncing: None, + job_token_scopes: None, }, ], same_worker: false, @@ -293,7 +296,7 @@ async fn test_deno_flow(db: Pool) -> anyhow::Result<()> { let port = server.addr.port(); for i in 0..50 { - println!("deno flow iteration: {}", i); + println!("bun flow iteration: {}", i); let job = run_job_in_new_worker_until_complete(&db, false, job.clone(), port).await; // println!("job: {:#?}", job.flow_status); let result = job.json_result().unwrap(); @@ -343,13 +346,13 @@ async fn test_identity(db: Pool) -> anyhow::Result<()> { #[cfg(feature = "deno_core")] #[sqlx::test(fixtures("base"))] -async fn test_deno_flow_same_worker(db: Pool) -> anyhow::Result<()> { +async fn test_bun_flow_same_worker(db: Pool) -> anyhow::Result<()> { initialize_tracing().await; let server = ApiServer::start(db.clone()).await?; let write_file = r#"export async function main(loop: boolean, i: number, path: string) { - await Deno.writeTextFile(`./shared/${path}`, `${loop} ${i}`); + await Bun.write(`./shared/${path}`, `${loop} ${i}`); }"# .to_string(); @@ -370,7 +373,7 @@ async fn test_deno_flow_same_worker(db: Pool) -> anyhow::Result<()> { ), ] .into(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: write_file.clone(), path: None, lock: None, @@ -398,6 +401,7 @@ async fn test_deno_flow_same_worker(db: Pool) -> anyhow::Result<()> { apply_preprocessor: None, pass_flow_input_directly: None, debouncing: None, + job_token_scopes: None, }, FlowModule { id: "b".to_string(), @@ -428,7 +432,7 @@ async fn test_deno_flow_same_worker(db: Pool) -> anyhow::Result<()> { ), ] .into(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: write_file, path: None, lock: None, @@ -455,6 +459,7 @@ async fn test_deno_flow_same_worker(db: Pool) -> anyhow::Result<()> { apply_preprocessor: None, pass_flow_input_directly: None, debouncing: None, + job_token_scopes: None, }, FlowModule { id: "e".to_string(), @@ -467,9 +472,9 @@ async fn test_deno_flow_same_worker(db: Pool) -> anyhow::Result<()> { InputTransform::Static { value: windmill_common::worker::to_raw_value(&"outer.txt") }, )] .into(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#"export async function main(path: string, path2: string) { - return await Deno.readTextFile(`./shared/${path}`) + "," + await Deno.readTextFile(`./shared/${path2}`); + return await Bun.file(`./shared/${path}`).text() + "," + await Bun.file(`./shared/${path2}`).text(); }"# .to_string(), path: None, @@ -498,6 +503,7 @@ async fn test_deno_flow_same_worker(db: Pool) -> anyhow::Result<()> { apply_preprocessor: None, pass_flow_input_directly: None, debouncing: None, + job_token_scopes: None, }, ], modules_node: None, @@ -520,6 +526,7 @@ async fn test_deno_flow_same_worker(db: Pool) -> anyhow::Result<()> { apply_preprocessor: None, pass_flow_input_directly: None, debouncing: None, + job_token_scopes: None, }, FlowModule { id: "c".to_string(), @@ -539,9 +546,9 @@ async fn test_deno_flow_same_worker(db: Pool) -> anyhow::Result<()> { ), ] .into(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#"export async function main(path: string, loops: string[], path2: string) { - return await Deno.readTextFile(`./shared/${path}`) + "," + loops + "," + await Deno.readTextFile(`./shared/${path2}`); + return await Bun.file(`./shared/${path}`).text() + "," + loops + "," + await Bun.file(`./shared/${path2}`).text(); }"# .to_string(), path: None, @@ -569,6 +576,7 @@ async fn test_deno_flow_same_worker(db: Pool) -> anyhow::Result<()> { apply_preprocessor: None, pass_flow_input_directly: None, debouncing: None, + job_token_scopes: None, }, ], same_worker: true, @@ -610,7 +618,7 @@ async fn test_same_worker_survives_empty_branch(db: Pool) -> anyhow::R "id": "c", "value": { "type": "rawscript", - "language": "deno", + "language": "bun", "content": "export function main(){ return 1 }", } }] @@ -622,7 +630,7 @@ async fn test_same_worker_survives_empty_branch(db: Pool) -> anyhow::R "id": "b", "value": { "type": "rawscript", - "language": "deno", + "language": "bun", "content": "export function main(){ return 42 }", } } @@ -663,7 +671,7 @@ async fn test_flow_result_by_id(db: Pool) -> anyhow::Result<()> { "id": "a", "value": { "type": "rawscript", - "language": "deno", + "language": "bun", "content": "export function main(){ return 42 }", } }, @@ -678,7 +686,7 @@ async fn test_flow_result_by_id(db: Pool) -> anyhow::Result<()> { "value": { "input_transforms": {"v": {"type": "javascript", "expr": "results.a"}}, "type": "rawscript", - "language": "deno", + "language": "bun", "content": "export function main(v){ return v }", } @@ -1723,6 +1731,286 @@ async fn test_postgresql_cached_connection_resets_session( Ok(()) } +/// The idle cached connection must not hold the server slot a job's own +/// connection needs. A role limited to one connection stands in for a +/// session-mode pooler with one slot, where the second job would wait forever +/// instead of failing. +#[sqlx::test(fixtures("base"))] +#[serial(pg_cache)] +async fn test_postgresql_cached_connection_released_for_other_key( + db: Pool, +) -> anyhow::Result<()> { + use windmill_worker::pg_executor::clear_pg_cache; + + initialize_tracing().await; + clear_pg_cache().await; + + sqlx::query( + "DO $$ BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'wm_pg_cache_one_conn') THEN + CREATE ROLE wm_pg_cache_one_conn LOGIN PASSWORD 'changeme' CONNECTION LIMIT 1; + END IF; + END $$", + ) + .execute(&db) + .await?; + + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + // Keys for the same role: one pool scope when only the sslmode differs, two + // when the database does, which the role's limit still counts together. + let run = |dbname: &str, sslmode: &str| { + RunJob::from(JobPayload::Code(RawCode { + hash: None, + content: "SELECT 1 as n;".into(), + path: None, + lock: None, + language: ScriptLang::Postgresql, + cache_ttl: None, + cache_ignore_s3_path: None, + dedicated_worker: None, + concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default( + ) + .into(), + debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), + modules: None, + tag: None, + })) + .arg( + "database", + json!({"host": "localhost", "port": 5432, "dbname": dbname, + "user": "wm_pg_cache_one_conn", "password": "changeme", "sslmode": sslmode}), + ) + .run_until_complete(&db, false, port) + }; + + // The evicted connection's backend exits asynchronously, so a fresh + // connection can briefly still count it. Retrying absorbs that; without the + // eviction the cached connection stays open for 60s and every retry fails. + for (dbname, sslmode) in [ + ("windmill", "disable"), + ("windmill", "prefer"), + ("postgres", "disable"), + ("windmill", "disable"), + ] { + let mut result = json!(null); + for _ in 0..5 { + result = run(dbname, sslmode).await.json_result().unwrap(); + if result == json!([{"n": 1}]) { + break; + } + tokio::time::sleep(std::time::Duration::from_millis(200)).await; + } + assert_eq!(result, json!([{"n": 1}]), "{dbname} sslmode={sslmode}"); + } + + clear_pg_cache().await; + Ok(()) +} + +/// A transaction a script leaves open must not carry over into the next job +/// that would reuse the connection. +#[sqlx::test(fixtures("base"))] +#[serial(pg_cache)] +async fn test_postgresql_open_transaction_not_reused(db: Pool) -> anyhow::Result<()> { + use windmill_worker::pg_executor::clear_pg_cache; + + initialize_tracing().await; + clear_pg_cache().await; + + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let dbname: String = sqlx::query_scalar("SELECT current_database()") + .fetch_one(&db) + .await?; + + let run = |content: &str| { + RunJob::from(JobPayload::Code(RawCode { + hash: None, + content: content.to_string(), + path: None, + lock: None, + language: ScriptLang::Postgresql, + cache_ttl: None, + cache_ignore_s3_path: None, + dedicated_worker: None, + concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default( + ) + .into(), + debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), + modules: None, + tag: None, + })) + .arg( + "database", + json!({"host": "localhost", "port": 5432, "dbname": dbname, "user": "postgres", "password": "changeme"}), + ) + .run_until_complete(&db, false, port) + }; + + run("SELECT 1 as n;").await.json_result().unwrap(); + let opened = run("BEGIN; SELECT 1 as n;").await.json_result().unwrap(); + assert_eq!(opened, json!([{"n": 1}])); + run("SELECT 2 as n;").await.json_result().unwrap(); + + // A connection kept in the cache with the transaction open would sit idle + // in it, holding its locks, and run the next job inside it. + let mut idle_in_transaction = -1; + for _ in 0..20 { + idle_in_transaction = sqlx::query_scalar::<_, i64>( + "SELECT count(*) FROM pg_stat_activity + WHERE datname = current_database() AND state LIKE 'idle in transaction%'", + ) + .fetch_one(&db) + .await?; + if idle_in_transaction == 0 { + break; + } + tokio::time::sleep(std::time::Duration::from_millis(100)).await; + } + assert_eq!(idle_in_transaction, 0); + + clear_pg_cache().await; + Ok(()) +} + +/// Closing the connection of a cancelled job does not stop its query: the server +/// only notices once it next writes to the client. +#[sqlx::test(fixtures("base"))] +#[serial(pg_cache)] +async fn test_postgresql_cancelled_job_stops_its_query(db: Pool) -> anyhow::Result<()> { + use windmill_worker::pg_executor::clear_pg_cache; + + initialize_tracing().await; + clear_pg_cache().await; + + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let dbname: String = sqlx::query_scalar("SELECT current_database()") + .fetch_one(&db) + .await?; + let still_running = "SELECT count(*) FROM pg_stat_activity + WHERE datname = current_database() AND state = 'active' + AND pid <> pg_backend_pid() AND query LIKE '%pg_sleep(30)%'"; + + let cancelled = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)); + let job = RunJob::from(JobPayload::Code(RawCode { + hash: None, + content: "SELECT pg_sleep(30);".to_string(), + path: None, + lock: None, + language: ScriptLang::Postgresql, + cache_ttl: None, + cache_ignore_s3_path: None, + dedicated_worker: None, + concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default() + .into(), + debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), + modules: None, + tag: None, + })) + .arg( + "database", + json!({"host": "localhost", "port": 5432, "dbname": dbname, "user": "postgres", "password": "changeme"}), + ) + .run_until_complete_with(&db, false, port, |id| { + let (db, cancelled) = (db.clone(), cancelled.clone()); + async move { + tokio::spawn(async move { + while sqlx::query_scalar::<_, i64>(still_running) + .fetch_one(&db) + .await + .unwrap() + == 0 + { + tokio::time::sleep(std::time::Duration::from_millis(50)).await; + } + sqlx::query("UPDATE v2_job_queue SET canceled_by = 'test-user' WHERE id = $1") + .bind(id) + .execute(&db) + .await + .unwrap(); + cancelled.store(true, std::sync::atomic::Ordering::Relaxed); + }); + } + }) + .await; + assert!(!job.success); + // Otherwise the job failed before its query ran, and there was nothing to stop. + assert!(cancelled.load(std::sync::atomic::Ordering::Relaxed)); + + let mut running = -1; + for _ in 0..50 { + running = sqlx::query_scalar::<_, i64>(still_running) + .fetch_one(&db) + .await?; + if running == 0 { + break; + } + tokio::time::sleep(std::time::Duration::from_millis(100)).await; + } + assert_eq!(running, 0); + + clear_pg_cache().await; + Ok(()) +} + +/// A worker alternating between two databases keeps a connection for each. +#[sqlx::test(fixtures("base"))] +#[serial(pg_cache)] +async fn test_postgresql_cache_keeps_a_connection_per_database( + db: Pool, +) -> anyhow::Result<()> { + use std::sync::atomic::Ordering; + use windmill_worker::pg_executor::{clear_pg_cache, CACHE_HITS}; + + initialize_tracing().await; + clear_pg_cache().await; + + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + let run = |dbname: &str| { + RunJob::from(JobPayload::Code(RawCode { + hash: None, + content: "SELECT current_database() as d;".into(), + path: None, + lock: None, + language: ScriptLang::Postgresql, + cache_ttl: None, + cache_ignore_s3_path: None, + dedicated_worker: None, + concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default( + ) + .into(), + debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), + modules: None, + tag: None, + })) + .arg( + "database", + json!({"host": "localhost", "port": 5432, "dbname": dbname, + "user": "postgres", "password": "changeme"}), + ) + .run_until_complete(&db, false, port) + }; + + let hits_before = CACHE_HITS.load(Ordering::Relaxed); + for dbname in ["windmill", "postgres", "windmill", "postgres"] { + let result = run(dbname).await.json_result().unwrap(); + assert_eq!(result, json!([{"d": dbname}])); + } + let hits = CACHE_HITS.load(Ordering::Relaxed) - hits_before; + assert_eq!( + hits, 2, + "the last two jobs should reuse a cached connection" + ); + + clear_pg_cache().await; + Ok(()) +} + /// Runs multiple PG jobs through a SINGLE worker (like production) to verify /// that SET ROLE / search_path changes do not leak across jobs. #[sqlx::test(fixtures("base"))] @@ -3569,7 +3857,7 @@ fn module_add_item_to_list(i: i32, id: &str) -> serde_json::Value { } }, "type": "rawscript", - "language": "deno", + "language": "bun", "content": "export function main(array, i){ array.push(i); return array }", } }) @@ -3580,7 +3868,7 @@ fn module_failure() -> serde_json::Value { "value": { "input_transforms": {}, "type": "rawscript", - "language": "deno", + "language": "bun", "content": "export function main(){ throw Error('failure') }", } }) @@ -3598,7 +3886,7 @@ async fn test_branchone_simple(db: Pool) -> anyhow::Result<()> { "id": "a", "value": { "type": "rawscript", - "language": "deno", + "language": "bun", "content": "export function main(){ return [1] }", } }, @@ -3636,7 +3924,7 @@ async fn test_branchone_with_cond(db: Pool) -> anyhow::Result<()> { "id": "a", "value": { "type": "rawscript", - "language": "deno", + "language": "bun", "content": "export function main(){ return [1] }", } }, @@ -3674,7 +3962,7 @@ async fn test_branchall_sequential(db: Pool) -> anyhow::Result<()> { "id": "a", "value": { "type": "rawscript", - "language": "deno", + "language": "bun", "content": "export function main(){ return [1] }", } }, @@ -3714,7 +4002,7 @@ async fn test_branchall_simple(db: Pool) -> anyhow::Result<()> { "id": "a", "value": { "type": "rawscript", - "language": "deno", + "language": "bun", "content": "export function main(){ return [1] }", } }, @@ -3762,7 +4050,7 @@ async fn test_branchall_skip_failure(db: Pool) -> anyhow::Result<()> { "id": "a", "value": { "type": "rawscript", - "language": "deno", + "language": "bun", "content": "export function main(){ return [1] }", } }, @@ -3798,7 +4086,7 @@ async fn test_branchall_skip_failure(db: Pool) -> anyhow::Result<()> { "id": "a", "value": { "type": "rawscript", - "language": "deno", + "language": "bun", "content": "export function main(){ return [1] }", } }, @@ -3844,7 +4132,7 @@ async fn test_branchone_nested(db: Pool) -> anyhow::Result<()> { "id": "a", "value": { "type": "rawscript", - "language": "deno", + "language": "bun", "content": "export function main(){ return [] }", } }, @@ -3903,7 +4191,7 @@ async fn test_branchall_nested(db: Pool) -> anyhow::Result<()> { "id": "a", "value": { "type": "rawscript", - "language": "deno", + "language": "bun", "content": "export function main(){ return [1] }", } }, @@ -3969,7 +4257,7 @@ async fn test_failure_module(db: Pool) -> anyhow::Result<()> { "n": { "type": "javascript", "expr": "flow_input.n", }, }, "type": "rawscript", - "language": "deno", + "language": "bun", "content": "export function main(n, l) { if (n == 0) throw Error(JSON.stringify(l)); return { l: [...l, 0] } }", }, }, { @@ -3980,7 +4268,7 @@ async fn test_failure_module(db: Pool) -> anyhow::Result<()> { "n": { "type": "javascript", "expr": "flow_input.n", }, }, "type": "rawscript", - "language": "deno", + "language": "bun", "content": "export function main(n, l) { if (n == 1) throw Error(JSON.stringify(l)); return { l: [...l, 1] } }", }, }, { @@ -3990,7 +4278,7 @@ async fn test_failure_module(db: Pool) -> anyhow::Result<()> { "n": { "type": "javascript", "expr": "flow_input.n", }, }, "type": "rawscript", - "language": "deno", + "language": "bun", "content": "export function main(n, l) { if (n == 2) throw Error(JSON.stringify(l)); return { l: [...l, 2] } }", }, }], @@ -3998,7 +4286,7 @@ async fn test_failure_module(db: Pool) -> anyhow::Result<()> { "value": { "input_transforms": { "error": { "type": "javascript", "expr": "previous_result", } }, "type": "rawscript", - "language": "deno", + "language": "bun", "content": "export function main(error) { return { 'from failure module': error } }", } }, @@ -4187,7 +4475,7 @@ fn hanging_step_value() -> serde_json::Value { serde_json::json!({ "input_transforms": {}, "type": "rawscript", - "language": "deno", + "language": "bun", "content": "export async function main() { await new Promise((r) => setTimeout(r, 600000)); }", }) } @@ -4199,7 +4487,7 @@ fn marker_failure_module() -> serde_json::Value { "value": { "input_transforms": { "error": { "type": "javascript", "expr": "previous_result", } }, "type": "rawscript", - "language": "deno", + "language": "bun", "content": "export function main(error) { return { handled_unrecoverable: true, error } }", } }) @@ -4297,7 +4585,7 @@ async fn test_unrecoverable_failure_on_continue_on_error_runs_failure_module( "value": { "input_transforms": {}, "type": "rawscript", - "language": "deno", + "language": "bun", "content": "export function main() { return { ran_b: true }; }", }, }, @@ -4336,7 +4624,7 @@ async fn test_run_wait_result_early_return_with_failure_module( "id": "a", "value": { "type": "rawscript", - "language": "deno", + "language": "bun", "input_transforms": {}, "content": "export function main() { throw new Error('boom'); }", }, @@ -4344,7 +4632,7 @@ async fn test_run_wait_result_early_return_with_failure_module( "failure_module": { "value": { "type": "rawscript", - "language": "deno", + "language": "bun", "input_transforms": {}, "content": "export function main() { return { recovered: true } }", }, @@ -4410,6 +4698,142 @@ async fn test_run_wait_result_early_return_with_failure_module( Ok(()) } +/// `recover: true` from the error handler of a step that fails inside a loop turns the flow +/// green, as it does for a top-level step, without changing which steps run: a loop that +/// stops at a failed iteration still stops there, one that skips failures still carries on. +#[cfg(feature = "deno_core")] +#[sqlx::test(fixtures("base"))] +async fn test_failure_module_recover_inside_loop(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + let failing_loop = |skip_failures: bool| { + json!({ + "id": "loop", + "value": { + "type": "forloopflow", + "iterator": { "type": "static", "value": [1, 2, 3] }, + "skip_failures": skip_failures, + "modules": [{ + "id": "a", + "value": { + "input_transforms": {}, + "type": "rawscript", + "language": "bun", + "content": "export function main() { throw new Error('boom'); }", + }, + }], + }, + }) + }; + let run = |loop_step: serde_json::Value| { + let flow: FlowValue = serde_json::from_value(json!({ + "modules": [loop_step, { + "id": "b", + "value": { + "input_transforms": {}, + "type": "rawscript", + "language": "bun", + "content": "export function main() { return { ran_b: true } }", + }, + }], + "failure_module": { + "value": { + "input_transforms": {}, + "type": "rawscript", + "language": "bun", + "content": "export function main() { return { handled: true, recover: true } }", + }, + }, + })) + .unwrap(); + RunJob::from(JobPayload::RawFlow { value: flow, path: None, restarted_from: None }) + .run_until_complete(&db, false, port) + }; + let handler_runs = || { + sqlx::query_scalar::<_, i64>( + "SELECT count(*) FROM v2_job_completed c JOIN v2_job j USING (id) + WHERE j.kind = 'preview' AND c.result @> '{\"handled\": true}'::jsonb", + ) + .fetch_one(&db) + }; + + let stopped = run(failing_loop(false)).await; + let stopped_handler_runs = handler_runs().await?; + let skipping = run(failing_loop(true)).await; + let skipping_handler_runs = handler_runs().await? - stopped_handler_runs; + // A parallel node counts its iterations' stored status: an iteration that ended on a + // recovered failure must still be stored as failed there. + let parallel = run(json!({ + "id": "par", + "value": { + "type": "forloopflow", + "iterator": { "type": "static", "value": [1, 2] }, + "parallel": true, + "skip_failures": false, + "modules": [failing_loop(false)], + }, + })) + .await; + // A recovery the inner loop absorbs, then an error raised starting the next step: that + // error is a new failure and must leave the flow failed. + #[cfg(feature = "quickjs")] + let chaining_error = run(json!({ + "id": "outer", + "value": { + "type": "forloopflow", + "iterator": { "type": "static", "value": [1] }, + "skip_failures": false, + "modules": [failing_loop(true), { + "id": "guarded", + "skip_if": { "expr": "missingFunction()" }, + "value": { "input_transforms": {}, "type": "identity" }, + }], + }, + })) + .await; + + server.close().await.unwrap(); + + #[cfg(feature = "quickjs")] + assert!( + !chaining_error.success, + "an error no handler recovered must keep the flow failed" + ); + + assert!( + stopped.success, + "a recovered failure inside a loop should end the flow as a success" + ); + assert!( + stopped.json_result().unwrap().get("ran_b").is_none(), + "the step after the loop must not run, as without recovery" + ); + assert_eq!( + stopped_handler_runs, 1, + "the loop must stop at the first failed iteration" + ); + + assert!(skipping.success); + assert_eq!( + skipping.json_result().unwrap(), + json!({ "ran_b": true }), + "a loop that skips failures runs every iteration and the following step" + ); + assert_eq!(skipping_handler_runs, 3); + + assert!( + !parallel + .json_result() + .unwrap() + .to_string() + .contains("ran_b"), + "the parallel loop fails as without recovery, so the step after it must not run" + ); + Ok(()) +} + #[cfg(feature = "python")] #[sqlx::test(fixtures("base"))] async fn test_flow_lock_all(db: Pool) -> anyhow::Result<()> { @@ -4576,6 +5000,101 @@ async fn test_flow_lock_all(db: Pool) -> anyhow::Result<()> { Ok(()) } +/// A deployed flow runs a multi-step loop body as the flow node its deploy stored, which +/// carries its own copy of the flow's `preserve_step_tags`. A preview never loads that node. +#[sqlx::test(fixtures("base"))] +async fn test_saved_flow_preserves_nested_step_tags(db: Pool) -> anyhow::Result<()> { + use futures::StreamExt; + + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let client = reqwest::Client::new(); + let flow = json!({ + "path": "g/all/saved_step_tags", "summary": "", "description": "", "tag": "python3", + "schema": { "type": "object", "properties": {}, "required": [] }, + "value": { + "preserve_step_tags": true, + "modules": [{ + "id": "loop", + "value": { + "type": "forloopflow", "skip_failures": false, + "iterator": { "type": "static", "value": [2, 3] }, + "modules": [{ + "id": "own_tag", + "value": { + "type": "rawscript", "language": "bun", "lock": "", "tag": "bun", + "content": "export function main(n: number) { return n * n; }", + "input_transforms": { + "n": { "type": "javascript", "expr": "flow_input.iter.value" } + } + } + }, { + "id": "untagged", + "value": { + "type": "rawscript", "language": "bun", "lock": "", + "content": "export function main(n: number) { return n + 1; }", + "input_transforms": { + "n": { "type": "javascript", "expr": "results.own_tag" } + } + } + }] + } + }] + } + }); + let mut completed = listen_for_completed_jobs(&db).await; + client + .post(format!( + "http://localhost:{port}/api/w/test-workspace/flows/create" + )) + .bearer_auth("SECRET_TOKEN") + .json(&flow) + .send() + .await? + .error_for_status()?; + let deployment = in_test_worker(&db, completed.next(), port).await.unwrap(); + let deployed = completed_job(deployment, &db).await; + assert!(deployed.success, "{:?}", deployed.result); + + let job_id = client + .post(format!( + "http://localhost:{port}/api/w/test-workspace/jobs/run/f/g/all/saved_step_tags" + )) + .bearer_auth("SECRET_TOKEN") + .json(&json!({})) + .send() + .await? + .error_for_status()? + .text() + .await? + .parse::()?; + in_test_worker(&db, completed.find(&job_id), port).await; + let job = completed_job(job_id, &db).await; + assert!(job.success, "{:?}", job.result); + let tags = sqlx::query_as::<_, (String, String)>( + "WITH RECURSIVE descendants AS ( + SELECT id FROM v2_job WHERE id = $1 + UNION ALL + SELECT child.id FROM v2_job child JOIN descendants ON child.parent_job = descendants.id + ) + SELECT DISTINCT flow_step_id, tag FROM v2_job + WHERE id IN (SELECT id FROM descendants) AND flow_step_id IN ('own_tag', 'untagged') + ORDER BY flow_step_id, tag", + ) + .bind(job_id) + .fetch_all(&db) + .await?; + assert_eq!( + tags, + vec![ + ("own_tag".to_owned(), "bun".to_owned()), + ("untagged".to_owned(), "python3".to_owned()) + ] + ); + Ok(()) +} + #[cfg(feature = "deno_core")] #[sqlx::test(fixtures("base"))] @@ -5276,7 +5795,7 @@ async fn test_job_labels(db: Pool) -> anyhow::Result<()> { console.log(greet) return { greet, wm_labels: ["yolo", "greet", "greet", world] }; }"#, - "language": "deno", + "language": "bun", "input_transforms": { "world": { "type": "javascript", "expr": "flow_input.world" } } @@ -5485,7 +6004,7 @@ async fn test_flow_substep_tag_availability_check(db: Pool) -> anyhow: "id": "a", "value": { "type": "rawscript", - "language": "deno", + "language": "bun", "content": "export function main() { return 42; }", "tag": "restricted-tag", }, @@ -5545,7 +6064,7 @@ async fn test_flow_substep_tag_checked_on_resolved_value(db: Pool) -> "id": id, "value": { "type": "rawscript", - "language": "deno", + "language": "bun", "content": "export function main() { return { lang: 'bun' } }", "tag": tag, }, @@ -5653,6 +6172,7 @@ async fn test_flow_tag_judged_as_written_before_preprocessor( apply_preprocessor: true, version: 1443253234253456, labels: None, + job_token_scopes: None, }) .as_user("test-user-2", "test2@windmill.dev") .run_until_complete(&db, false, port) diff --git a/backend/tests/worker_group_pause.rs b/backend/tests/worker_group_pause.rs new file mode 100644 index 0000000000..b528bbfd77 --- /dev/null +++ b/backend/tests/worker_group_pause.rs @@ -0,0 +1,71 @@ +//! In its own test binary: it flips the process-wide `WORKER_CONFIG`, which would stall every +//! other worker test sharing the process. + +use futures::StreamExt; +use sqlx::{Pool, Postgres}; +use std::time::Duration; +use windmill_common::{ + jobs::{JobPayload, RawCode}, + scripts::ScriptLang, + worker::{Connection, WORKER_CONFIG}, +}; +use windmill_test_utils::*; + +fn set_paused(paused: bool) { + let mut wc = (**WORKER_CONFIG.load()).clone(); + wc.paused = paused; + WORKER_CONFIG.store(std::sync::Arc::new(wc)); +} + +#[sqlx::test(fixtures("base"))] +async fn paused_worker_leaves_jobs_queued_until_resumed(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + set_paused(true); + let id = RunJob::from(JobPayload::Code(RawCode { + hash: None, + content: "echo done".to_string(), + path: None, + lock: None, + language: ScriptLang::Bash, + cache_ttl: None, + cache_ignore_s3_path: None, + dedicated_worker: None, + concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default() + .into(), + debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), + modules: None, + tag: None, + })) + .push(&db) + .await; + + let mut completed = listen_for_completed_jobs(&db).await; + in_test_worker( + Connection::Sql(db.clone()), + async { + tokio::time::sleep(Duration::from_secs(3)).await; + let running = + sqlx::query_scalar::<_, bool>("SELECT running FROM v2_job_queue WHERE id = $1") + .bind(id) + .fetch_optional(&db) + .await + .unwrap(); + assert_eq!( + running, + Some(false), + "a paused worker must not pull the job" + ); + + set_paused(false); + while completed.next().await != Some(id) {} + }, + port, + ) + .await; + + assert!(completed_job(id, &db).await.success); + Ok(()) +} diff --git a/backend/tests/worker_ping_ip.rs b/backend/tests/worker_ping_ip.rs index 772f4193c3..2421e888ef 100644 --- a/backend/tests/worker_ping_ip.rs +++ b/backend/tests/worker_ping_ip.rs @@ -2,10 +2,19 @@ use sqlx::{Pool, Postgres}; use uuid::Uuid; use windmill_common::{ external_ip::UNKNOWN_IP, - worker::{insert_ping_query, update_worker_ping_main_loop_query}, + worker::{insert_ping_query, set_worker_draining_query, update_worker_ping_main_loop_query}, }; async fn insert_ping(db: &Pool, worker: &str, ip: Option<&str>) -> anyhow::Result<()> { + register(db, worker, ip, None).await +} + +async fn register( + db: &Pool, + worker: &str, + ip: Option<&str>, + draining: Option, +) -> anyhow::Result<()> { insert_ping_query( "test-instance", worker, @@ -19,12 +28,33 @@ async fn insert_ping(db: &Pool, worker: &str, ip: Option<&str>) -> any None, None, false, + draining, db, ) .await?; Ok(()) } +/// A worker re-registers mid-life once its external IP resolves. That registration must not +/// clear a draining flag the same process already published; only a worker starting up under +/// a reclaimed name resets it. +#[sqlx::test] +async fn only_a_starting_worker_resets_draining(db: Pool) -> anyhow::Result<()> { + let draining = || async { + sqlx::query_scalar::<_, bool>("SELECT draining FROM worker_ping WHERE worker = 'wk-drain'") + .fetch_one(&db) + .await + }; + register(&db, "wk-drain", None, Some(false)).await?; + set_worker_draining_query("wk-drain", &db).await?; + register(&db, "wk-drain", Some("1.2.3.4"), None).await?; + assert!(draining().await?); + + register(&db, "wk-drain", None, Some(false)).await?; + assert!(!draining().await?); + Ok(()) +} + /// The external IP resolves in the background, so the initial ping often has none yet. That must /// not blank the address a previous process wrote to the row this one reclaims — worker names are /// stable across restarts under EXIT_AFTER_N_JOBS. diff --git a/backend/tests/zombie_flow_recovery.rs b/backend/tests/zombie_flow_recovery.rs index fb658dfb03..333aac70a8 100644 --- a/backend/tests/zombie_flow_recovery.rs +++ b/backend/tests/zombie_flow_recovery.rs @@ -78,7 +78,7 @@ async fn test_between_steps_zombie_restart_reuses_all_children( "id": "inner", "value": { "type": "rawscript", - "language": "deno", + "language": "bun", "input_transforms": { "v": { "type": "javascript", "expr": "flow_input.iter.value" } }, @@ -91,7 +91,7 @@ async fn test_between_steps_zombie_restart_reuses_all_children( "id": "after", "value": { "type": "rawscript", - "language": "deno", + "language": "bun", "input_transforms": { "loop_res": { "type": "javascript", "expr": "results.fanout" } }, @@ -210,7 +210,7 @@ async fn test_mid_iteration_zombie_not_reused(db: Pool) -> anyhow::Res "id": "inner", "value": { "type": "rawscript", - "language": "deno", + "language": "bun", "input_transforms": { "v": { "type": "javascript", "expr": "flow_input.iter.value" } }, @@ -223,7 +223,7 @@ async fn test_mid_iteration_zombie_not_reused(db: Pool) -> anyhow::Res "id": "after", "value": { "type": "rawscript", - "language": "deno", + "language": "bun", "input_transforms": { "loop_res": { "type": "javascript", "expr": "results.fanout" } }, @@ -331,7 +331,7 @@ async fn test_nested_restart_not_swallowed_by_zombie_reuse( { "id": "inner_first", "value": { - "type": "rawscript", "language": "deno", + "type": "rawscript", "language": "bun", "input_transforms": {}, "content": "export function main() { return 'first' }" } @@ -339,7 +339,7 @@ async fn test_nested_restart_not_swallowed_by_zombie_reuse( { "id": "inner_second", "value": { - "type": "rawscript", "language": "deno", + "type": "rawscript", "language": "bun", "input_transforms": { "first": { "type": "javascript", "expr": "results.inner_first" } }, @@ -353,7 +353,7 @@ async fn test_nested_restart_not_swallowed_by_zombie_reuse( { "id": "after", "value": { - "type": "rawscript", "language": "deno", + "type": "rawscript", "language": "bun", "input_transforms": { "b": { "type": "javascript", "expr": "results.branch" } }, @@ -466,7 +466,7 @@ async fn test_raw_flow_restart_does_not_reuse_edited_step( "modules": [{ "id": "inner", "value": { - "type": "rawscript", "language": "deno", + "type": "rawscript", "language": "bun", "input_transforms": { "v": { "type": "javascript", "expr": "flow_input.iter.value" } }, @@ -478,7 +478,7 @@ async fn test_raw_flow_restart_does_not_reuse_edited_step( { "id": "after", "value": { - "type": "rawscript", "language": "deno", + "type": "rawscript", "language": "bun", "input_transforms": { "loop_res": { "type": "javascript", "expr": "results.fanout" } }, @@ -578,7 +578,7 @@ async fn test_non_monitor_cancel_is_not_reused(db: Pool) -> anyhow::Re "modules": [{ "id": "inner", "value": { - "type": "rawscript", "language": "deno", + "type": "rawscript", "language": "bun", "input_transforms": { "v": { "type": "javascript", "expr": "flow_input.iter.value" } }, @@ -590,7 +590,7 @@ async fn test_non_monitor_cancel_is_not_reused(db: Pool) -> anyhow::Re { "id": "after", "value": { - "type": "rawscript", "language": "deno", + "type": "rawscript", "language": "bun", "input_transforms": { "loop_res": { "type": "javascript", "expr": "results.fanout" } }, diff --git a/backend/windmill-ai/Cargo.toml b/backend/windmill-ai/Cargo.toml index b1dea3ae54..44c4ac83c7 100644 --- a/backend/windmill-ai/Cargo.toml +++ b/backend/windmill-ai/Cargo.toml @@ -28,6 +28,7 @@ eventsource-stream.workspace = true futures.workspace = true http.workspace = true mime_guess.workspace = true +rand.workspace = true reqwest.workspace = true serde.workspace = true serde_json.workspace = true diff --git a/backend/windmill-ai/src/ai_bedrock.rs b/backend/windmill-ai/src/ai_bedrock.rs index a9318a1f3a..f55fcca415 100644 --- a/backend/windmill-ai/src/ai_bedrock.rs +++ b/backend/windmill-ai/src/ai_bedrock.rs @@ -477,6 +477,29 @@ pub async fn assume_role_with_oidc_token( // Error Formatting // ============================================================================ +/// The error for a Bedrock stream that failed after it opened, retrying the exceptions the +/// Vercel AI SDK retries (`amazon-bedrock-stream-error.ts`) and a broken connection. The +/// SDK's own retries cover the opening call alone. +pub fn bedrock_stream_error( + error: &aws_sdk_bedrockruntime::error::SdkError< + aws_sdk_bedrockruntime::types::error::ConverseStreamOutputError, + R, + >, +) -> Error { + let message = format!("Bedrock stream error: {}", format_bedrock_error(error)); + match error.as_service_error() { + Some(e) + if !(e.is_internal_server_exception() + || e.is_model_stream_error_exception() + || e.is_service_unavailable_exception() + || e.is_throttling_exception()) => + { + Error::AIError(message) + } + _ => crate::retry::transient_error(message, None), + } +} + /// Format AWS SDK errors with detailed information pub fn format_bedrock_error(error: &aws_sdk_bedrockruntime::error::SdkError) -> String where diff --git a/backend/windmill-ai/src/ai_google.rs b/backend/windmill-ai/src/ai_google.rs index bb8ae83e13..9f48edca8a 100644 --- a/backend/windmill-ai/src/ai_google.rs +++ b/backend/windmill-ai/src/ai_google.rs @@ -326,6 +326,17 @@ pub struct GeminiSSEEvent { pub candidates: Option>, #[serde(rename = "usageMetadata")] pub usage_metadata: Option, + #[serde(default)] + pub error: Option, + #[serde(rename = "promptFeedback", default)] + pub prompt_feedback: Option, +} + +/// Set when Gemini refuses the prompt itself, in which case no candidate follows. +#[derive(Deserialize, Debug)] +pub struct GeminiPromptFeedback { + #[serde(rename = "blockReason", default)] + pub block_reason: Option, } // ============================================================================ @@ -550,10 +561,40 @@ pub fn openai_tools_to_gemini( } } +/// Follows a Gemini stream to tell a complete turn from one cut short. +#[derive(Default)] +pub struct GeminiTurn { + ended: bool, + has_output: bool, +} + +impl GeminiTurn { + pub fn observe(&mut self, event: &GeminiParsedEvent) { + // A function call arrives whole and may come without a `finishReason`, so it + // counts too, as in gemini-cli's stream validation. + self.ended |= event.finish_reason.is_some() || !event.tool_calls.is_empty(); + self.has_output |= event.text.is_some() || event.reasoning.is_some(); + } + + /// Whether the turn's final event arrived. + pub fn ended(&self) -> bool { + self.ended + } + + /// Whether a stream that closed cleanly here holds the whole turn. Gemini 2.5 closes + /// its stream without a single event when thinking spends the whole + /// `maxOutputTokens`: an empty answer, which a retry would only get again, rather + /// than a cut one. + pub fn complete_at_close(&self) -> bool { + self.ended || !self.has_output + } +} + /// Parse one Gemini SSE data line into a [`GeminiParsedEvent`]. /// /// Returns `Ok(None)` for empty data or unrecognised payloads (e.g. `"[DONE]"`). /// Logs a warning and returns `Ok(None)` on JSON parse errors rather than propagating. +/// An error the stream reports is returned as one. pub fn parse_gemini_sse_event(data: &str) -> Result, Error> { if data.is_empty() || data == "[DONE]" { return Ok(None); @@ -567,6 +608,21 @@ pub fn parse_gemini_sse_event(data: &str) -> Result, E } }; + if let Some(error) = event.error { + return Err(error.into_error("Gemini")); + } + + // `BLOCK_REASON_UNSPECIFIED` is the protobuf enum's placeholder, not a block. + if let Some(reason) = event + .prompt_feedback + .and_then(|f| f.block_reason) + .filter(|r| !matches!(r.as_str(), "" | "BLOCK_REASON_UNSPECIFIED")) + { + return Err(Error::AIError(format!( + "Gemini blocked the prompt: {reason}" + ))); + } + let mut parsed = GeminiParsedEvent { usage: event.usage_metadata, ..Default::default() }; let Some(candidates) = event.candidates else { diff --git a/backend/windmill-ai/src/ai_providers.rs b/backend/windmill-ai/src/ai_providers.rs index a4dcbc4f35..3b3af833d6 100644 --- a/backend/windmill-ai/src/ai_providers.rs +++ b/backend/windmill-ai/src/ai_providers.rs @@ -39,6 +39,7 @@ lazy_static::lazy_static! { pub const OPENAI_BASE_URL: &str = "https://api.openai.com/v1"; pub const DEEPSEEK_BASE_URL: &str = "https://api.deepseek.com/v1"; pub const GOOGLE_AI_BASE_URL: &str = "https://generativelanguage.googleapis.com/v1beta"; +pub const TYPESAFE_BASE_URL: &str = "https://api.typesafe.ai/v1"; /// Hosts that serve the OpenAI API with Azure conventions: Azure OpenAI /// (`*.openai.azure.com`), AI Foundry (`*.services.ai.azure.com`, @@ -122,9 +123,41 @@ pub enum AIProvider { CustomAI, #[serde(rename = "aws_bedrock")] AWSBedrock, + /// TypeSafe's Jev decision model. It answers typed questions rather than chatting, so only + /// an AI decision runs it (`run_decision`), never the agent loop. + TypeSafe, + /// Cloudflare Workers AI, for its Jev-compatible decision models (Clef). Decisions only, as + /// for TypeSafe; its base URL comes from the resource's account id. + Cloudflare, } +/// The Workers AI models URL of a Cloudflare account. The id goes into the path, so it is held to +/// the characters an account id has rather than trusted to stay inside its segment. +pub fn cloudflare_workers_ai_base_url(account_id: Option<&str>) -> Result { + match account_id { + Some(id) if !id.is_empty() && id.chars().all(|c| c.is_ascii_alphanumeric()) => Ok(format!( + "https://api.cloudflare.com/client/v4/accounts/{id}/ai/run/@cf/cloudflare" + )), + Some(_) => Err(Error::BadRequest( + "The Cloudflare account_id must be letters and digits only".to_string(), + )), + None => Err(Error::BadRequest( + "A Cloudflare resource needs an account_id".to_string(), + )), + } +} impl AIProvider { + /// A provider that answers decisions (typed questions) and serves no chat. + pub fn is_decision_provider(&self) -> bool { + matches!(self, AIProvider::TypeSafe | AIProvider::Cloudflare) + } + + /// A provider an AI decision runs on: the decision providers, and OpenAI, whose Decisions + /// API sits next to its chat routes on the same resource. + pub fn runs_decisions(&self) -> bool { + self.is_decision_provider() || matches!(self, AIProvider::OpenAI) + } + /// Get the base URL for the AI provider pub async fn get_base_url(&self, resource_base_url: Option, db: &DB) -> Result { if let Some(base_url) = resource_base_url { @@ -177,6 +210,10 @@ impl AIProvider { AIProvider::TogetherAI => Ok("https://api.together.xyz/v1".to_string()), AIProvider::Anthropic => Ok("https://api.anthropic.com/v1".to_string()), AIProvider::Mistral => Ok("https://api.mistral.ai/v1".to_string()), + AIProvider::TypeSafe => Ok(TYPESAFE_BASE_URL.to_string()), + AIProvider::Cloudflare => Err(Error::BadRequest( + "A Cloudflare resource needs an account_id".to_string(), + )), p @ (AIProvider::CustomAI | AIProvider::AzureOpenAI | AIProvider::AzureFoundry) => { Err(Error::BadRequest(format!( "{:?} provider requires a base URL in the resource", diff --git a/backend/windmill-ai/src/decision.rs b/backend/windmill-ai/src/decision.rs new file mode 100644 index 0000000000..b3a35dab2a --- /dev/null +++ b/backend/windmill-ai/src/decision.rs @@ -0,0 +1,556 @@ +//! AI decision steps: one call to a decision endpoint, which answers typed questions (`choice`, +//! `score`, `noul`) about a state with calibrated probabilities instead of text. The questions and +//! answers are System One's (TypeSafe's Jev, Cloudflare's Jev-compatible Clef on Workers AI); +//! OpenAI's Decisions API is asked and read through a translation to that shape. + +use std::time::Duration; + +use serde::{Deserialize, Serialize}; +use serde_json::{value::RawValue, Map, Value}; +use windmill_common::error::{Error, Result}; + +use crate::{ + ai_providers::AIProvider, + credentials::ProviderCredentials, + proxy::{common_outbound_headers, retain_effective_credentials}, + types::{ProviderWithResource, TokenUsage}, + utils::pinned_ai_client_for, +}; + +/// The inputs of an AI decision step or tool, as its `input_transforms` resolve them. +#[derive(Deserialize, Debug)] +pub struct AIDecisionArgs { + pub provider: ProviderWithResource, + #[serde(default)] + pub state: Option, + #[serde(default)] + pub questions: Option, +} + +/// The alias TypeSafe keeps on its current model, sent when the step names none. +pub const TYPESAFE_DEFAULT_MODEL: &str = "jev-latest"; +/// Cloudflare's larger decision model, run when the step names none. +pub const CLOUDFLARE_DEFAULT_MODEL: &str = "clef"; +/// The model OpenAI's Decisions API serves, run when the step names none. +pub const OPENAI_DEFAULT_MODEL: &str = "gpt-6-luna"; + +#[derive(Serialize)] +struct SystemOneRequest<'a> { + model: &'a str, + state: &'a Value, + questions: &'a Map, +} + +#[derive(Deserialize)] +struct SystemOneResponse { + #[serde(default)] + model: Option, + answers: Box, + #[serde(default)] + usage: Option, +} + +/// Workers AI wraps the System One response in its API envelope. +#[derive(Deserialize)] +struct CloudflareEnvelope { + result: SystemOneResponse, +} + +#[derive(Deserialize)] +struct SystemOneUsage { + input_tokens: Option, + output_tokens: Option, +} + +/// A decision's result. `output` holds the answers keyed by question name, where an agent's +/// holds its answer, so a later step reads both as `results..output`. +#[derive(Serialize)] +pub struct DecisionResult { + pub output: Box, + /// The version that answered, which `jev-latest` resolves to. + #[serde(skip_serializing_if = "Option::is_none")] + pub model: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub usage: Option, +} + +/// The state and questions of a decision, refused here rather than by the endpoint when +/// either is missing or of a shape TypeSafe rejects: a flow that passes an empty expression result +/// should read what it forgot, without a request going out. +pub fn decision_inputs<'a>( + state: Option<&'a Value>, + questions: Option<&'a Value>, +) -> Result<(&'a Value, &'a Map)> { + let state = match state { + None | Some(Value::Null) => { + return Err(Error::BadRequest( + "'state' must be provided for an AI decision".to_string(), + )) + } + Some(Value::String(s)) if s.trim().is_empty() => { + return Err(Error::BadRequest( + "'state' must be provided for an AI decision".to_string(), + )) + } + Some(state @ (Value::String(_) | Value::Object(_) | Value::Array(_))) => state, + Some(_) => { + return Err(Error::BadRequest( + "'state' must be a text, an object or an array for an AI decision".to_string(), + )) + } + }; + let questions = match questions { + Some(Value::Object(questions)) if !questions.is_empty() => questions, + _ => { + return Err(Error::BadRequest( + "'questions' must be an object naming at least one question for an AI decision" + .to_string(), + )) + } + }; + Ok((state, questions)) +} + +/// Posts a decision request with the resource's bearer key and returns the response body. +async fn post_decision( + credentials: &ProviderCredentials, + endpoint: &str, + vendor: &str, + payload: &T, + timeout: Duration, +) -> Result { + let auth_headers = retain_effective_credentials( + credentials, + vec![( + "Authorization", + format!( + "Bearer {}", + credentials.api_key.as_deref().unwrap_or_default() + ), + )], + ); + + // The endpoint derives from the resource's base URL, so the connect is pinned to the + // address the SSRF check validated. + let client = pinned_ai_client_for(&credentials.base_url).await?; + let mut request = client.post(endpoint).timeout(timeout); + for (name, value) in auth_headers { + request = request.header(name, value); + } + for (name, value) in common_outbound_headers(credentials) { + request = request.header(name, value); + } + + let response = request + .json(payload) + .send() + .await + .map_err(|e| Error::ExecutionErr(format!("Failed to call {vendor}: {e}")))?; + + let status = response.status(); + if !status.is_success() { + let body = response + .text() + .await + .unwrap_or_else(|_| "".to_string()); + return Err(Error::ExecutionErr(format!( + "{vendor} error calling {endpoint}: {status} - {body}" + ))); + } + + response + .text() + .await + .map_err(|e| Error::ExecutionErr(format!("Failed to read the {vendor} response: {e}"))) +} + +/// Answers a decision on the endpoint its provider serves. +pub async fn run_decision( + credentials: &ProviderCredentials, + model: &str, + state: &Value, + questions: &Map, + timeout: Duration, +) -> Result { + if credentials.provider == AIProvider::OpenAI { + run_openai_decisions(credentials, model, state, questions, timeout).await + } else { + run_systemone(credentials, model, state, questions, timeout).await + } +} + +#[derive(Serialize)] +struct OpenAIDecisionsRequest<'a> { + model: &'a str, + input: Value, + questions: Vec, +} + +#[derive(Deserialize)] +struct OpenAIDecisionsResponse { + #[serde(default)] + model: Option, + answers: Vec>, + #[serde(default)] + usage: Option, +} + +/// The state as a Decisions `input`. OpenAI reads a text or a list of user messages, so a list of +/// user messages (the only way to pass an image) goes as it is and any other object or array, +/// a chat history included, as its JSON text. +fn openai_input(state: &Value) -> Value { + match state { + Value::String(_) => state.clone(), + Value::Array(items) if !items.is_empty() && items.iter().all(is_openai_user_message) => { + state.clone() + } + _ => Value::String(state.to_string()), + } +} + +/// Exactly a message the Decisions API reads: `role: "user"` with a text, or with parts that are +/// each an `input_text` or an `input_image`. +fn is_openai_user_message(message: &Value) -> bool { + let is_part = |part: &Value| match part.get("type").and_then(Value::as_str) { + Some("input_text") => part.get("text").is_some_and(Value::is_string), + Some("input_image") => part.get("image_url").is_some_and(Value::is_string), + _ => false, + }; + message.get("role").and_then(Value::as_str) == Some("user") + && match message.get("content") { + Some(Value::String(_)) => true, + Some(Value::Array(parts)) => !parts.is_empty() && parts.iter().all(is_part), + _ => false, + } +} + +/// The questions as OpenAI's Decisions API takes them: a list naming each question, with a +/// `noul` asked as its `predicate` and each score level as a label. +fn openai_questions(questions: &Map) -> Result> { + let text = |v: &Value| match v { + Value::String(s) => s.clone(), + v => v.to_string(), + }; + // A structured description goes as its JSON text, as structured instructions do. + let stated = |description: Option<&Value>| match description { + None | Some(Value::Null) => None, + Some(Value::String(d)) if d.trim().is_empty() => None, + Some(d) => Some(text(d)), + }; + questions + .iter() + .map(|(name, question)| { + let criteria = question.get("criteria"); + let mut instructions = question.get("instructions").map(text).unwrap_or_default(); + let mut asked = Map::new(); + let kind = match question.get("type").and_then(Value::as_str) { + Some("choice") => { + let choices = criteria + .and_then(Value::as_object) + .filter(|c| !c.is_empty()) + .ok_or_else(|| { + Error::BadRequest(format!( + "The choice question '{name}' needs its options as 'criteria'" + )) + })? + .iter() + .map(|(value, d)| match stated(Some(d)) { + Some(d) => serde_json::json!({"value": value, "description": d}), + None => serde_json::json!({"value": value}), + }) + .collect(); + asked.insert("choices".to_string(), Value::Array(choices)); + "choice" + } + Some("score") => { + let levels = criteria + .and_then(Value::as_array) + .filter(|c| !c.is_empty()) + .ok_or_else(|| { + Error::BadRequest(format!( + "The score question '{name}' needs its levels as 'criteria'" + )) + })? + .iter() + .map(|level| serde_json::json!({"label": text(level)})) + .collect(); + asked.insert("levels".to_string(), Value::Array(levels)); + "score" + } + Some("noul") => { + for (key, meaning) in [("true", "Yes"), ("false", "No")] { + if let Some(d) = stated(criteria.and_then(|c| c.get(key))) { + instructions.push_str(&format!("\n{meaning}: {d}")); + } + } + "predicate" + } + other => { + return Err(Error::BadRequest(format!( + "The question '{name}' has type {}, which is not 'choice', 'score' or 'noul'", + other.map_or("none".to_string(), |t| format!("'{t}'")) + ))) + } + }; + asked.insert("type".to_string(), kind.into()); + asked.insert("name".to_string(), name.as_str().into()); + asked.insert("instructions".to_string(), instructions.into()); + Ok(Value::Object(asked)) + }) + .collect() +} + +/// OpenAI's answers keyed by question name, as every decision's are. A predicate's probability is +/// also given as `noul`, so a flow reads a yes/no answer the same way whichever provider gave it. +/// A refused question fails the decision: left in the output it has no `choice` or `noul`, so a +/// branch on it would quietly take its "no" or default arm. +fn openai_answers(answers: Vec>) -> Result> { + let mut by_name = Map::new(); + for mut answer in answers { + let Some(Value::String(name)) = answer.remove("name") else { + continue; + }; + match answer.get("type").and_then(Value::as_str) { + Some("refusal") => { + return Err(Error::ExecutionErr(format!( + "OpenAI refused to answer the question '{name}': {}", + Value::Object(answer) + ))) + } + Some("predicate") => { + if let Some(probability) = answer.get("probability").cloned() { + answer.insert("noul".to_string(), probability); + } + } + _ => {} + } + by_name.insert(name, Value::Object(answer)); + } + Ok(by_name) +} + +async fn run_openai_decisions( + credentials: &ProviderCredentials, + model: &str, + state: &Value, + questions: &Map, + timeout: Duration, +) -> Result { + let model = match model.trim() { + "" => OPENAI_DEFAULT_MODEL, + model => model, + }; + let endpoint = format!("{}/decisions", credentials.base_url.trim_end_matches('/')); + let body = post_decision( + credentials, + &endpoint, + "OpenAI", + &OpenAIDecisionsRequest { + model, + input: openai_input(state), + questions: openai_questions(questions)?, + }, + timeout, + ) + .await?; + let parsed = serde_json::from_str::(&body) + .map_err(|e| Error::ExecutionErr(format!("Unexpected OpenAI response ({e}): {body}")))?; + + Ok(DecisionResult { + output: serde_json::value::to_raw_value(&openai_answers(parsed.answers)?) + .map_err(|e| Error::internal_err(format!("serializing decision answers: {e}")))?, + model: parsed.model, + usage: parsed + .usage + .map(|u| TokenUsage::from_input_output(u.input_tokens, u.output_tokens)) + .filter(|u| !u.is_empty()), + }) +} + +async fn run_systemone( + credentials: &ProviderCredentials, + model: &str, + state: &Value, + questions: &Map, + timeout: Duration, +) -> Result { + let cloudflare = credentials.provider == AIProvider::Cloudflare; + let model = match model.trim() { + "" if cloudflare => CLOUDFLARE_DEFAULT_MODEL, + "" => TYPESAFE_DEFAULT_MODEL, + model => model, + }; + let base = credentials.base_url.trim_end_matches('/'); + // Workers AI serves each model at its own URL; the model goes into the path, so it is held to + // the characters a model id has. + let endpoint = if cloudflare { + if !model + .chars() + .all(|c| c.is_ascii_alphanumeric() || matches!(c, '-' | '_' | '.')) + { + return Err(Error::BadRequest(format!( + "Invalid Cloudflare decision model: {model}" + ))); + } + format!("{base}/{model}") + } else { + format!("{base}/systemone") + }; + let vendor = if cloudflare { "Cloudflare" } else { "TypeSafe" }; + let body = post_decision( + credentials, + &endpoint, + vendor, + &SystemOneRequest { model, state, questions }, + timeout, + ) + .await?; + // Tried in turn rather than as an untagged enum, which cannot hold the raw `answers`. + let parsed = match serde_json::from_str::(&body) { + Ok(parsed) => parsed, + Err(e) => match serde_json::from_str::(&body) { + Ok(envelope) => envelope.result, + Err(_) => { + return Err(Error::ExecutionErr(format!( + "Unexpected {vendor} response ({e}): {body}" + ))) + } + }, + }; + + Ok(DecisionResult { + output: parsed.answers, + model: parsed.model, + // Dropped when empty, as an agent's is, so `usage` is either counted or absent. + usage: parsed + .usage + .map(|u| TokenUsage::from_input_output(u.input_tokens, u.output_tokens)) + .filter(|u| !u.is_empty()), + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + /// The account id goes into the URL path, so anything that could leave its segment is refused. + #[test] + fn cloudflare_account_id_stays_in_its_segment() { + use crate::ai_providers::cloudflare_workers_ai_base_url; + assert_eq!( + cloudflare_workers_ai_base_url(Some("abc123")).unwrap(), + "https://api.cloudflare.com/client/v4/accounts/abc123/ai/run/@cf/cloudflare" + ); + for id in [None, Some("a/b"), Some("../x"), Some("a?b"), Some("")] { + assert!(cloudflare_workers_ai_base_url(id).is_err(), "{id:?}"); + } + } + + /// A `noul` goes out as a predicate with its yes/no meanings in the instructions, and comes + /// back readable as `noul`; choices and levels go out as lists, answers come back by name. + #[test] + fn openai_questions_and_answers_translate() { + let questions = json!({ + "intent": {"type": "choice", "instructions": "Why?", "criteria": {"refund": {"when": "paid"}, "bug": ""}}, + "urgency": {"type": "score", "instructions": "How urgent?", "criteria": ["Can wait", "Now"]}, + "angry": {"type": "noul", "instructions": "Angry?", "criteria": {"true": "Insults"}} + }); + let asked = Value::Array(openai_questions(questions.as_object().unwrap()).unwrap()); + assert_eq!( + asked, + json!([ + {"choices": [ + {"value": "refund", "description": "{\"when\":\"paid\"}"}, + {"value": "bug"} + ], "type": "choice", "name": "intent", "instructions": "Why?"}, + {"levels": [ + {"label": "Can wait"}, + {"label": "Now"} + ], "type": "score", "name": "urgency", "instructions": "How urgent?"}, + {"type": "predicate", "name": "angry", "instructions": "Angry?\nYes: Insults"} + ]) + ); + for bad in [ + json!({"q": {"type": "choice", "instructions": "?"}}), + json!({"q": {"type": "score", "instructions": "?", "criteria": []}}), + json!({"q": {"type": "predicate", "instructions": "?"}}), + ] { + assert!(openai_questions(bad.as_object().unwrap()).is_err(), "{bad}"); + } + + let answers = serde_json::from_value(json!([ + {"type": "predicate", "name": "angry", "probability": 0.92}, + {"type": "choice", "name": "intent", "choice": "refund", "confidence": 0.9} + ])) + .unwrap(); + assert_eq!( + Value::Object(openai_answers(answers).unwrap()), + json!({ + "angry": {"type": "predicate", "probability": 0.92, "noul": 0.92}, + "intent": {"type": "choice", "choice": "refund", "confidence": 0.9} + }) + ); + + let messages = json!([{"role": "user", "content": [{"type": "input_text", "text": "hi"}]}]); + assert_eq!(openai_input(&messages), messages); + assert_eq!(openai_input(&json!({"m": "hi"})), json!("{\"m\":\"hi\"}")); + assert_eq!( + openai_input(&json!([{"role": "admin"}])), + json!("[{\"role\":\"admin\"}]") + ); + for records in [ + json!([{"role": "user", "content": {"ticket": "charged twice"}}]), + json!([{"role": "user", "content": null}]), + json!([{"role": "user", "content": ["a"]}]), + json!([{"role": "user", "content": [{"type": "input_text", "text": {"t": 1}}]}]), + json!([{"role": "user", "content": [{"type": "input_image"}]}]), + ] { + assert_eq!(openai_input(&records), json!(records.to_string())); + } + assert_eq!( + openai_input(&json!([{"role": "user", "content": "Hi"}])), + json!([{"role": "user", "content": "Hi"}]) + ); + // A chat history is evaluated as text: OpenAI takes user messages only. + let history = + json!([{"role": "user", "content": "Hi"}, {"role": "assistant", "content": "Yo"}]); + assert_eq!(openai_input(&history), json!(history.to_string())); + let refused = + serde_json::from_value(json!([{"type": "refusal", "name": "angry"}])).unwrap(); + assert!(openai_answers(refused).is_err()); + } + + /// Refused are the values a flow produces when it forgot the input (no key, `null`, a blank + /// string from an empty expression) and the scalars TypeSafe rejects. A text, an object or an + /// array is a state, whatever it holds. + #[test] + fn decision_inputs_refuse_what_typesafe_would() { + let questions = json!({"urgent": {"type": "noul", "instructions": "Is it urgent?"}}); + for state in [ + None, + Some(json!(null)), + Some(json!(" ")), + Some(json!(false)), + Some(json!(0)), + ] { + let err = decision_inputs(state.as_ref(), Some(&questions)).unwrap_err(); + assert!(err.to_string().contains("'state'"), "{err}"); + } + for state in [json!("0"), json!({}), json!([]), json!({"m": "hi"})] { + assert!( + decision_inputs(Some(&state), Some(&questions)).is_ok(), + "{state}" + ); + } + for questions in [ + None, + Some(json!({})), + Some(json!([])), + Some(json!("urgent")), + ] { + let err = decision_inputs(Some(&json!("hi")), questions.as_ref()).unwrap_err(); + assert!(err.to_string().contains("'questions'"), "{err}"); + } + } +} diff --git a/backend/windmill-ai/src/image_handler.rs b/backend/windmill-ai/src/image_handler.rs index 5bd9794da5..a4f360cdb4 100644 --- a/backend/windmill-ai/src/image_handler.rs +++ b/backend/windmill-ai/src/image_handler.rs @@ -47,6 +47,9 @@ pub async fn upload_image_to_s3( }) } +// Bounds each attachment's download before base64 expansion; provider limits may be lower. +const MAX_S3_MEDIA_BYTES: usize = 30 * 1024 * 1024; + /// Download an S3 image and convert it to a base64 data URL. /// /// The caller must provide an AuthedClient authorized for `workspace_id`. @@ -57,7 +60,12 @@ pub async fn download_and_encode_s3_image( ) -> Result<(String, String), Error> { // Download the image from S3 let image_bytes = client - .download_s3_file(workspace_id, &image.s3, image.storage.clone()) + .download_s3_file( + workspace_id, + &image.s3, + image.storage.clone(), + Some(MAX_S3_MEDIA_BYTES), + ) .await .map_err(|e| Error::internal_err(format!("Failed to download S3 image: {}", e)))?; diff --git a/backend/windmill-ai/src/lib.rs b/backend/windmill-ai/src/lib.rs index e81d030ed3..634614a3cf 100644 --- a/backend/windmill-ai/src/lib.rs +++ b/backend/windmill-ai/src/lib.rs @@ -5,11 +5,13 @@ pub mod ai_google; pub mod ai_providers; pub mod ai_types; pub mod credentials; +pub mod decision; pub mod image_handler; pub mod model_context; pub mod providers; pub mod proxy; pub mod query_builder; +pub mod retry; pub mod sse; pub mod types; pub mod utils; diff --git a/backend/windmill-ai/src/providers/bedrock.rs b/backend/windmill-ai/src/providers/bedrock.rs index 836244a666..5aa142a127 100644 --- a/backend/windmill-ai/src/providers/bedrock.rs +++ b/backend/windmill-ai/src/providers/bedrock.rs @@ -9,11 +9,12 @@ use super::{anthropic_model_rejects_sampling_params, REASONING_OFF_SENTINEL}; use crate::{ ai_bedrock::{ - bedrock_model_supports_prompt_caching, bedrock_stream_event_is_block_stop, - bedrock_stream_event_to_reasoning_delta, bedrock_stream_event_to_text, - bedrock_stream_event_to_tool_delta, bedrock_stream_event_to_tool_delta_with_block_index, - bedrock_stream_event_to_tool_start, bedrock_stream_event_to_tool_start_with_block_index, - build_tool_config, create_inference_config, format_bedrock_error, json_to_document, + bedrock_model_supports_prompt_caching, bedrock_stream_error, + bedrock_stream_event_is_block_stop, bedrock_stream_event_to_reasoning_delta, + bedrock_stream_event_to_text, bedrock_stream_event_to_tool_delta, + bedrock_stream_event_to_tool_delta_with_block_index, bedrock_stream_event_to_tool_start, + bedrock_stream_event_to_tool_start_with_block_index, build_tool_config, + create_inference_config, format_bedrock_error, json_to_document, openai_messages_to_bedrock, streaming_tool_calls_to_openai, BearerTokenProvider, BedrockClient, StreamingToolCall, }, @@ -24,6 +25,7 @@ use crate::{ image_handler::prepare_messages_for_api, proxy::ProxyBuildArgs, query_builder::{ParsedResponse, StreamEventSink}, + retry::{openai_stream_error_chunk, truncated_stream_error}, types::{OpenAIMessage, StreamingEvent, TokenUsage, ToolDef}, }; use bytes::Bytes; @@ -461,26 +463,31 @@ pub fn sdk_stream_to_sse( async_stream::stream! { let mut stream = stream; let mut state = BedrockSseStreamState::new(id, model, created); + let mut message_stopped = false; - loop { + let failure = loop { match stream.recv().await { Ok(Some(event)) => { + if matches!( + event, + aws_sdk_bedrockruntime::types::ConverseStreamOutput::MessageStop(_) + ) { + message_stopped = true; + } for chunk in bedrock_sse_chunks_for_event(&event, &mut state) { yield Ok(chunk); } } - Ok(None) => break, - Err(e) => { - yield Err(std::io::Error::new( - std::io::ErrorKind::Other, - e.to_string(), - )); - break; - } + Ok(None) => break (!message_stopped).then(truncated_stream_error), + Err(_) if message_stopped => break None, + Err(e) => break Some(bedrock_stream_error(&e)), } - } + }; - yield Ok(Bytes::from("data: [DONE]\n\n")); + match failure { + Some(error) => yield Ok(Bytes::from(openai_stream_error_chunk(&error))), + None => yield Ok(Bytes::from("data: [DONE]\n\n")), + } } } @@ -1081,11 +1088,19 @@ impl BedrockQueryBuilder { // Claude reasoning block for the turn (only populated when thinking is on), // attached to the first tool call for replay before toolUse. let mut reasoning: Option = None; + let mut message_stopped = false; // Process stream events using shared parsing functions loop { match stream.recv().await { Ok(Some(event)) => { + if matches!( + event, + aws_sdk_bedrockruntime::types::ConverseStreamOutput::MessageStop(_) + ) { + message_stopped = true; + } + // Fold reasoning deltas into the turn's reasoning block (for // replay before toolUse, required by Claude when thinking is // on) and stream the readable summary as a thinking affordance. @@ -1161,11 +1176,13 @@ impl BedrockQueryBuilder { } } Ok(None) => break, - Err(e) => { - return Err(Error::internal_err(format!("Bedrock stream error: {}", e))); - } + Err(_) if message_stopped => break, + Err(e) => return Err(bedrock_stream_error(&e)), } } + if !message_stopped { + return Err(truncated_stream_error()); + } // Send tool call events to stream processor if let Some(processor) = stream_event_sink.as_ref() { diff --git a/backend/windmill-ai/src/providers/google_ai.rs b/backend/windmill-ai/src/providers/google_ai.rs index 97d0383ba4..bbf45e5e57 100644 --- a/backend/windmill-ai/src/providers/google_ai.rs +++ b/backend/windmill-ai/src/providers/google_ai.rs @@ -5,17 +5,20 @@ use crate::{ parse_gemini_response, parse_gemini_sse_event, sanitize_schema_for_google, GeminiFunctionDeclaration, GeminiGenerationConfig, GeminiImageContent, GeminiImageRequest, GeminiImageResponse, GeminiInlineData, GeminiPart, GeminiPredictContent, GeminiTextRequest, - GeminiThinkingConfig, GeminiTool, + GeminiThinkingConfig, GeminiTool, GeminiTurn, }, image_handler::{download_and_encode_s3_image, prepare_messages_for_api}, proxy::{ProxyBuildArgs, ProxyRequest}, query_builder::{BuildRequestArgs, ParsedResponse, QueryBuilder, StreamEventSink}, + retry::{ + openai_stream_error_chunk, send_with_retries, transient_error, truncated_stream_error, + }, sse::{GeminiSSEParser, SSEParser}, types::*, }; use async_trait::async_trait; use bytes::Bytes; -use eventsource_stream::Eventsource; +use eventsource_stream::{EventStreamError, Eventsource}; use futures::{stream::BoxStream, StreamExt}; use http::{header, HeaderMap, HeaderValue, Method, StatusCode}; use serde::Deserialize; @@ -336,9 +339,7 @@ pub async fn handle_google_ai_chat_proxy( ) -> Result { let GoogleAIProxyRequest { request, model, stream } = build_google_ai_chat_proxy_request(args)?; - let response = - send_google_ai_proxy_request(client, request, "Failed to send request to Gemini API") - .await?; + let response = send_google_ai_proxy_request(client, request).await?; if stream { Ok(convert_streaming_response(response, &model)) @@ -356,8 +357,7 @@ pub async fn handle_google_ai_models_proxy( args: &ProxyBuildArgs<'_>, ) -> Result { let request = build_google_ai_models_proxy_request(args); - let response = - send_google_ai_proxy_request(client, request, "Failed to fetch Gemini models").await?; + let response = send_google_ai_proxy_request(client, request).await?; let gemini_resp: GeminiModelsResponse = response.json().await.map_err(|e| { Error::internal_err(format!("Failed to parse Gemini models response: {}", e)) @@ -505,26 +505,13 @@ fn add_google_ai_auth_header(headers: &mut Vec<(String, String)>, api_key: &str, async fn send_google_ai_proxy_request( client: &reqwest::Client, proxy_request: ProxyRequest, - send_error_message: &str, ) -> Result { let mut request = client.request(proxy_request.method.clone(), &proxy_request.url); for (header_name, header_value) in &proxy_request.headers { request = request.header(header_name.as_str(), header_value.as_str()); } - let response = request - .body(proxy_request.body) - .send() - .await - .map_err(|e| Error::internal_err(format!("{}: {}", send_error_message, e)))?; - - if let Err(e) = response.error_for_status_ref() { - let status = e.status().map(|s| s.to_string()).unwrap_or_default(); - let body = response.text().await.unwrap_or_default(); - return Err(Error::AIError(format!("{}: {}", status, body))); - } - - Ok(response) + send_with_retries(request.body(proxy_request.body)).await } fn convert_streaming_response(response: reqwest::Response, model: &str) -> GoogleAIProxyResponse { @@ -535,10 +522,15 @@ fn convert_streaming_response(response: reqwest::Response, model: &str) -> Googl let openai_sse_stream = async_stream::stream! { tokio::pin!(gemini_sse_stream); let mut tool_call_index: usize = 0; - while let Some(event) = gemini_sse_stream.next().await { + let mut turn = GeminiTurn::default(); + let failure = loop { + let Some(event) = gemini_sse_stream.next().await else { + break (!turn.complete_at_close()).then(truncated_stream_error); + }; match event { Ok(event) => match parse_gemini_sse_event(&event.data) { Ok(Some(parsed)) => { + turn.observe(&parsed); for chunk in gemini_event_to_openai_sse_chunks( &parsed, &id, &model, &mut tool_call_index, ) { @@ -546,12 +538,23 @@ fn convert_streaming_response(response: reqwest::Response, model: &str) -> Googl } } Ok(None) => {} - Err(e) => tracing::error!("Error parsing Gemini SSE event: {}", e), + Err(e) => break Some(e), }, + Err(EventStreamError::Transport(_)) if turn.ended() => break None, + Err(EventStreamError::Transport(e)) => { + break Some(transient_error( + format!("The connection to Gemini broke off mid-response: {e}"), + None, + )); + } Err(e) => tracing::error!("Error reading Gemini SSE stream: {}", e), } - } - yield Ok::(Bytes::from("data: [DONE]\n\n")); + }; + let last = match failure { + Some(error) => openai_stream_error_chunk(&error), + None => "data: [DONE]\n\n".to_string(), + }; + yield Ok::(Bytes::from(last)); } .boxed(); diff --git a/backend/windmill-ai/src/providers/mod.rs b/backend/windmill-ai/src/providers/mod.rs index b5b6281264..7dcddc93a3 100644 --- a/backend/windmill-ai/src/providers/mod.rs +++ b/backend/windmill-ai/src/providers/mod.rs @@ -62,9 +62,13 @@ const REASONING_RULES: &[ReasoningRule] = &[ can_disable: false, completions_tools_need_off: false, }, - // Live-verified: astra takes low..max only, where sol and luna also take `none`. + // Live-verified: gpt-6-astra and gpt-6.1-sol take low..max only, where gpt-6-sol and + // gpt-6-luna also take `none`. ReasoningRule { - matches: |m| base_id(m).starts_with("gpt-6-astra"), + matches: |m| { + let base = base_id(m); + base.starts_with("gpt-6-astra") || base.starts_with("gpt-6.1-sol") + }, can_disable: false, completions_tools_need_off: true, }, @@ -204,6 +208,12 @@ pub fn create_query_builder( AIProvider::AzureFoundry if AIProvider::is_anthropic_model(model) => Box::new( AnthropicQueryBuilder::new(AIProvider::AzureFoundry, AIPlatform::Standard), ), + // Only for forwarding through the API proxy, which is bearer auth on `/` as + // TypeSafe and Cloudflare expect. Neither serves a chat route for its decision models, and + // `run_agent` refuses both before a chat request could be built. + AIProvider::TypeSafe | AIProvider::Cloudflare => { + Box::new(OtherQueryBuilder::new(credentials.provider.clone())) + } _ => Box::new(OtherQueryBuilder::new(credentials.provider.clone())), } } diff --git a/backend/windmill-ai/src/providers/other.rs b/backend/windmill-ai/src/providers/other.rs index 1ea54a7e79..fea8edbdac 100644 --- a/backend/windmill-ai/src/providers/other.rs +++ b/backend/windmill-ai/src/providers/other.rs @@ -266,6 +266,7 @@ impl QueryBuilder for OtherQueryBuilder { mut events_str, stream_event_processor, usage: openai_usage, + .. } = openai_sse_parser; // Process streaming events with error handling diff --git a/backend/windmill-ai/src/proxy.rs b/backend/windmill-ai/src/proxy.rs index 3af8f88a1d..887cab0f3e 100644 --- a/backend/windmill-ai/src/proxy.rs +++ b/backend/windmill-ai/src/proxy.rs @@ -72,7 +72,9 @@ pub fn proxy_execution_mode(provider: &AIProvider) -> ProxyExecutionMode { | AIProvider::Groq | AIProvider::OpenRouter | AIProvider::TogetherAI - | AIProvider::CustomAI => ProxyExecutionMode::HttpForward, + | AIProvider::CustomAI + | AIProvider::TypeSafe + | AIProvider::Cloudflare => ProxyExecutionMode::HttpForward, AIProvider::GoogleAI => ProxyExecutionMode::NativeGoogleAi, AIProvider::AWSBedrock => ProxyExecutionMode::NativeAwsBedrock, } @@ -444,6 +446,8 @@ mod tests { (AIProvider::OpenRouter, ProxyExecutionMode::HttpForward), (AIProvider::TogetherAI, ProxyExecutionMode::HttpForward), (AIProvider::CustomAI, ProxyExecutionMode::HttpForward), + (AIProvider::TypeSafe, ProxyExecutionMode::HttpForward), + (AIProvider::Cloudflare, ProxyExecutionMode::HttpForward), (AIProvider::GoogleAI, ProxyExecutionMode::NativeGoogleAi), (AIProvider::AWSBedrock, ProxyExecutionMode::NativeAwsBedrock), ]; diff --git a/backend/windmill-ai/src/query_builder.rs b/backend/windmill-ai/src/query_builder.rs index 986c433f42..a08d01dadd 100644 --- a/backend/windmill-ai/src/query_builder.rs +++ b/backend/windmill-ai/src/query_builder.rs @@ -1,5 +1,7 @@ +use std::borrow::Cow; + use async_trait::async_trait; -use windmill_common::{client::AuthedClient, error::Error}; +use windmill_common::{client::AuthedClient, error::Error, utils::strip_json_nul}; use windmill_types::s3::S3Object; use crate::ai_types::OpenAIToolCall; @@ -92,6 +94,39 @@ pub enum ParsedResponse { }, } +impl ParsedResponse { + /// Drops every U+0000 the model emitted. Postgres stores neither a NUL in `text` nor its + /// `\u0000` escape in `jsonb`, and this output is written to both: tool job args, the flow + /// status, agent memory, conversation rows. + pub fn without_nul(mut self) -> Self { + fn strip(s: &mut String) { + if s.contains('\0') { + s.retain(|c| c != '\0'); + } + } + if let ParsedResponse::Text { content, reasoning, tool_calls, annotations, .. } = &mut self + { + for text in [content, reasoning].into_iter().flatten() { + strip(text); + } + for annotation in annotations { + strip(&mut annotation.url); + annotation.title.as_mut().into_iter().for_each(strip); + } + for tool_call in tool_calls { + strip(&mut tool_call.id); + strip(&mut tool_call.function.name); + // Arguments are serialized JSON, where a NUL is spelled as its escape. + strip(&mut tool_call.function.arguments); + if let Cow::Owned(stripped) = strip_json_nul(&tool_call.function.arguments) { + tool_call.function.arguments = stripped; + } + } + } + self + } +} + /// Trait for streaming AI events to a sink (e.g., database persistence). /// Implemented by the worker's StreamEventProcessor. #[async_trait] diff --git a/backend/windmill-ai/src/retry.rs b/backend/windmill-ai/src/retry.rs new file mode 100644 index 0000000000..9de9d309da --- /dev/null +++ b/backend/windmill-ai/src/retry.rs @@ -0,0 +1,474 @@ +use std::time::Duration; + +use reqwest::{header::HeaderMap, RequestBuilder, Response, StatusCode}; +use windmill_common::error::{to_anyhow, Error}; + +/// How many times a provider call is retried after its first attempt. +pub const MAX_RETRIES: u32 = 3; +const BASE_DELAY: Duration = Duration::from_secs(1); +const MAX_BACKOFF: Duration = Duration::from_secs(10); +/// A provider that asks for a longer wait is out of capacity for longer than anyone should +/// block on (a spent daily quota, say), so its error is returned instead of waited out. +const MAX_RETRY_AFTER: Duration = Duration::from_secs(60); + +/// A provider call that failed for a reason a later attempt may not meet: the provider was +/// overloaded, throttled or unreachable, or its response broke off part way. +#[derive(Debug)] +pub struct TransientAIError { + pub message: String, + /// The wait the provider asked for, when it named one. + pub retry_after: Option, +} + +impl std::fmt::Display for TransientAIError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(&self.message) + } +} + +impl std::error::Error for TransientAIError {} + +pub fn transient_error(message: impl Into, retry_after: Option) -> Error { + to_anyhow(TransientAIError { message: message.into(), retry_after }).into() +} + +pub fn as_transient(error: &Error) -> Option<&TransientAIError> { + match error { + Error::Anyhow { error, .. } => error.downcast_ref::(), + _ => None, + } +} + +/// The error to report once a transient failure is no longer retried, without the source +/// location `Error::Anyhow` renders. +pub fn into_final_error(error: Error) -> Error { + match as_transient(&error) { + Some(transient) => Error::AIError(transient.message.clone()), + None => error, + } +} + +/// The statuses the OpenAI and Anthropic SDKs and the Vercel AI SDK retry. +fn is_retryable_status(status: u16) -> bool { + matches!(status, 408 | 409 | 429 | 500..=599) +} + +/// The error for a provider's error response. The rules follow the OpenAI and Anthropic +/// SDKs (`x-should-retry`, then the status) and gemini-cli (a daily quota is spent until +/// tomorrow, and the wait Gemini asks for is in its body). +pub fn response_error( + status: StatusCode, + headers: &HeaderMap, + body: &str, + message: String, +) -> Error { + let retryable = match headers.get("x-should-retry").and_then(|v| v.to_str().ok()) { + Some("true") => true, + Some("false") => false, + _ => { + is_retryable_status(status.as_u16()) + // OpenAI answers a spent balance with the 429 it also uses for rate limits. + && !body.contains("insufficient_quota") + && !google_quota_spent_for_the_day(body) + } + }; + if retryable { + transient_error( + message, + retry_after(headers).or_else(|| google_retry_delay(body)), + ) + } else { + Error::AIError(message) + } +} + +/// The `details` of a Google API error body, where Gemini puts its quota and retry hints. +fn google_error_details(body: &str) -> Vec { + serde_json::from_str::(body) + .ok() + .and_then(|v| v.get("error")?.get("details")?.as_array().cloned()) + .unwrap_or_default() +} + +fn google_quota_spent_for_the_day(body: &str) -> bool { + google_error_details(body).iter().any(|detail| { + detail + .get("violations") + .and_then(|v| v.as_array()) + .is_some_and(|violations| { + violations.iter().any(|violation| { + violation + .get("quotaId") + .and_then(|id| id.as_str()) + .is_some_and(|id| id.contains("PerDay") || id.contains("Daily")) + }) + }) + }) +} + +/// The `RetryInfo.retryDelay` of a Google API error, a protobuf duration such as `"53s"`. +fn google_retry_delay(body: &str) -> Option { + google_error_details(body).iter().find_map(|detail| { + let delay = detail.get("retryDelay")?.as_str()?.strip_suffix('s')?; + let seconds = delay + .parse::() + .ok() + .filter(|v| v.is_finite() && *v >= 0.0)?; + Some(Duration::try_from_secs_f64(seconds).unwrap_or(Duration::MAX)) + }) +} + +/// The wait a provider asked for in its `retry-after-ms` or `retry-after` header. An +/// HTTP-date `retry-after` is ignored, which falls back to the computed backoff. +fn retry_after(headers: &HeaderMap) -> Option { + let seconds = |name: &str, scale: f64| { + headers + .get(name)? + .to_str() + .ok()? + .trim() + .parse::() + .ok() + .filter(|v| v.is_finite() && *v >= 0.0) + .map(|v| Duration::try_from_secs_f64(v / scale).unwrap_or(Duration::MAX)) + }; + seconds("retry-after-ms", 1000.0).or_else(|| seconds("retry-after", 1.0)) +} + +/// The error object a provider sends inside a stream: OpenAI-compatible chunks carry +/// `type` and a string or status `code`, Anthropic `type`, Gemini a status `code` and +/// `status`. +#[derive(Debug, Default, serde::Deserialize)] +pub struct StreamErrorBody { + #[serde(default)] + pub message: Option, + #[serde(default, rename = "type")] + pub kind: Option, + #[serde(default)] + pub code: Option, + #[serde(default)] + pub status: Option, +} + +impl StreamErrorBody { + /// The error for a failure an OpenAI-compatible or Gemini stream reported after opening + /// with a 200, classified as the Vercel AI SDK classifies OpenAI stream errors + /// (`openai-stream-error.ts`): the status `code` holds, else the status its code and type + /// name, else a server error. + pub fn into_error(self, provider: &str) -> Error { + let code = self.code_str(); + let discriminator = [code.as_deref(), self.kind.as_deref()] + .into_iter() + .flatten() + .collect::>() + .join(" ") + .to_lowercase(); + let has = |terms: &[&str]| terms.iter().any(|term| discriminator.contains(term)); + let status = match code.as_deref().and_then(|code| code.parse::().ok()) { + Some(status @ 400..=599) => status, + _ if has(&["insufficient_quota", "rate_limit"]) => 429, + _ if has(&["authentication"]) => 401, + _ if has(&["permission"]) => 403, + _ if has(&["not_found"]) => 404, + _ if has(&["invalid", "bad_request", "context_length"]) => 400, + _ if has(&["overload"]) => 503, + _ if has(&["timeout"]) => 504, + _ => 500, + }; + let retryable = !has(&["insufficient_quota"]) && is_retryable_status(status); + self.into_classified_error(provider, retryable) + } + + /// The error for a failure an Anthropic stream reported after opening with a 200. The + /// kinds retried are those the Vercel AI SDK retries, plus `timeout_error`, which + /// Anthropic documents as its 504; any other kind is a fault in the request or account. + pub fn into_anthropic_error(self) -> Error { + let retryable = matches!( + self.kind.as_deref(), + Some("api_error" | "overloaded_error" | "rate_limit_error" | "timeout_error") + ); + self.into_classified_error("Anthropic", retryable) + } + + fn code_str(&self) -> Option { + match self.code.as_ref()? { + serde_json::Value::String(code) => Some(code.clone()), + serde_json::Value::Number(code) => Some(code.to_string()), + _ => None, + } + } + + fn into_classified_error(self, provider: &str, retryable: bool) -> Error { + let kind = [self.kind.clone(), self.code_str(), self.status] + .into_iter() + .flatten() + .collect::>() + .join("/"); + let message = format!( + "{provider} stream error{}: {}", + (!kind.is_empty()) + .then(|| format!(" ({kind})")) + .unwrap_or_default(), + self.message.as_deref().unwrap_or("no details") + ); + if retryable { + transient_error(message, None) + } else { + Error::AIError(message) + } + } +} + +/// The error for a stream that closed before the provider sent the event ending a +/// complete response: cut off by the provider or by a proxy in between. +pub fn truncated_stream_error() -> Error { + transient_error( + "The AI provider's response stream ended before the response was complete", + None, + ) +} + +/// An OpenAI-format stream chunk reporting `error`, which ends a response converted to that +/// format after it started streaming: the chat's OpenAI SDK throws on a chunk carrying +/// `error`, where a bare end of stream would read as a finished answer. +pub fn openai_stream_error_chunk(error: &Error) -> String { + let message = as_transient(error).map_or_else(|| error.to_string(), |t| t.message.clone()); + format!( + "data: {}\n\n", + serde_json::json!({ "error": { "message": message } }) + ) +} + +/// Sends a provider request, retrying a transient failure while nothing of the response +/// has been read: a failed connection or an error status. Once retries are spent the error +/// carries the provider's answer, as a single attempt's would. +pub async fn send_with_retries(request: RequestBuilder) -> Result { + let mut backoff = Backoff::default(); + loop { + // A streamed body cannot be replayed, so such a request is sent once. + let Some(attempt) = request.try_clone() else { + return send_once(request).await.map_err(into_final_error); + }; + let error = match send_once(attempt).await { + Ok(response) => return Ok(response), + Err(error) => error, + }; + let Some(delay) = as_transient(&error).and_then(|t| backoff.next_delay(t)) else { + return Err(into_final_error(error)); + }; + tracing::warn!( + "AI provider call failed ({error}); retrying in {:.1}s ({}/{MAX_RETRIES})", + delay.as_secs_f64(), + backoff.retries() + ); + tokio::time::sleep(delay).await; + } +} + +/// The error for a provider request that failed before any response arrived. The request +/// timeout is the caller's whole budget, so a timed-out call leaves no time for another +/// attempt; a connect timeout has its own, shorter one. +pub fn send_error(e: reqwest::Error, context: &str) -> Error { + let message = format!("{context}: {e}"); + if e.is_builder() || (e.is_timeout() && !e.is_connect()) { + Error::AIError(message) + } else { + transient_error(message, None) + } +} + +async fn send_once(request: RequestBuilder) -> Result { + let response = request + .send() + .await + .map_err(|e| send_error(e, "Failed to reach the AI provider"))?; + let status = response.status(); + if status.is_client_error() || status.is_server_error() { + let headers = response.headers().clone(); + let body = response.text().await.unwrap_or_default(); + return Err(response_error(status, &headers, &body, body.clone())); + } + Ok(response) +} + +/// Counts the retries of one provider call and spaces them out. +#[derive(Debug, Default)] +pub struct Backoff { + retries: u32, +} + +impl Backoff { + /// The wait before retrying a call that failed with `error`, or `None` once it should + /// fail instead. + pub fn next_delay(&mut self, error: &TransientAIError) -> Option { + if self.retries >= MAX_RETRIES { + return None; + } + let delay = match error.retry_after { + Some(wait) if wait > MAX_RETRY_AFTER => return None, + // A zero wait would retry back to back, so it gets the computed backoff. + Some(wait) if !wait.is_zero() => wait, + // Jittered, so callers throttled together do not come back together. + _ => (BASE_DELAY * 2u32.pow(self.retries)) + .min(MAX_BACKOFF) + .mul_f64(0.5 + rand::random::() * 0.5), + }; + self.retries += 1; + Some(delay) + } + + /// Retries granted so far. + pub fn retries(&self) -> u32 { + self.retries + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Error responses as providers send them, each with the retry decision of the client + /// the rule is taken from. + #[test] + fn classifies_error_responses() { + const GEMINI_QUOTA: &str = r#"{"error":{"code":429,"status":"RESOURCE_EXHAUSTED","details":[{"@type":"type.googleapis.com/google.rpc.QuotaFailure","violations":[{"quotaId":"QUOTA_ID"}]},{"@type":"type.googleapis.com/google.rpc.RetryInfo","retryDelay":"53s"}]}}"#; + let cases: &[(u16, &[(&str, &str)], &str, Option>)] = &[ + ( + 429, + &[], + r#"{"error":{"code":"rate_limit_exceeded"}}"#, + Some(None), + ), + ( + 429, + &[], + r#"{"error":{"type":"insufficient_quota","code":"insufficient_quota"}}"#, + None, + ), + ( + 529, + &[], + r#"{"type":"error","error":{"type":"overloaded_error"}}"#, + Some(None), + ), + (501, &[], "", Some(None)), + (400, &[], "", None), + (401, &[], "", None), + (400, &[("x-should-retry", "true")], "", Some(None)), + (503, &[("x-should-retry", "false")], "", None), + (429, &[("retry-after", "2")], "", Some(Some(2))), + ( + 429, + &[], + &GEMINI_QUOTA.replace("QUOTA_ID", "GenerateRequestsPerMinutePerProjectPerModel"), + Some(Some(53)), + ), + ( + 429, + &[], + &GEMINI_QUOTA.replace( + "QUOTA_ID", + "GenerateRequestsPerDayPerProjectPerModel-FreeTier", + ), + None, + ), + ]; + for (status, headers, body, expected) in cases { + let headers = headers + .iter() + .map(|(k, v)| (k.parse().unwrap(), v.parse().unwrap())) + .collect::(); + let error = response_error( + StatusCode::from_u16(*status).unwrap(), + &headers, + body, + String::new(), + ); + let got = as_transient(&error).map(|t| t.retry_after.map(|d| d.as_secs())); + assert_eq!(&got, expected, "{status} {headers:?} {body}"); + } + } + + /// In-stream error objects as providers send them, after the 200 opened the stream. + #[test] + fn classifies_stream_errors() { + let retried = |json: &str, anthropic: bool| { + let body: StreamErrorBody = serde_json::from_str(json).unwrap(); + let error = if anthropic { + body.into_anthropic_error() + } else { + body.into_error("AI provider") + }; + as_transient(&error).is_some() + }; + for (json, expected) in [ + (r#"{"message":"m","type":"server_error"}"#, true), + ( + r#"{"message":"m","type":"requests","code":"rate_limit_exceeded"}"#, + true, + ), + ( + r#"{"message":"m","type":"insufficient_quota","code":"insufficient_quota"}"#, + false, + ), + ( + r#"{"message":"m","type":"invalid_request_error","code":"context_length_exceeded"}"#, + false, + ), + // LiteLLM's proxy puts the upstream status in `code` as a string. + (r#"{"message":"m","type":"None","code":"400"}"#, false), + (r#"{"message":"m","code":"503"}"#, true), + (r#"{"code":503,"message":"m","status":"UNAVAILABLE"}"#, true), + ( + r#"{"code":400,"message":"m","status":"INVALID_ARGUMENT"}"#, + false, + ), + (r#"{"message":"m"}"#, true), + ] { + assert_eq!(retried(json, false), expected, "{json}"); + } + for (kind, expected) in [ + ("overloaded_error", true), + ("api_error", true), + ("rate_limit_error", true), + ("timeout_error", true), + ("billing_error", false), + ("invalid_request_error", false), + ("some_new_error", false), + ] { + let json = format!(r#"{{"type":"{kind}","message":"m"}}"#); + assert_eq!(retried(&json, true), expected, "{kind}"); + } + } + + #[test] + fn honours_retry_after_up_to_a_bound() { + let mut headers = HeaderMap::new(); + headers.insert("retry-after", "2".parse().unwrap()); + assert_eq!(retry_after(&headers), Some(Duration::from_secs(2))); + headers.insert("retry-after-ms", "1500".parse().unwrap()); + assert_eq!(retry_after(&headers), Some(Duration::from_millis(1500))); + headers.remove("retry-after-ms"); + headers.insert("retry-after", "1e30".parse().unwrap()); + assert_eq!(retry_after(&headers), Some(Duration::MAX)); + + let mut backoff = Backoff::default(); + let asked = |secs| TransientAIError { + message: String::new(), + retry_after: Some(Duration::from_secs(secs)), + }; + assert_eq!(backoff.next_delay(&asked(5)), Some(Duration::from_secs(5))); + assert_eq!(backoff.next_delay(&asked(3600)), None); + } + + #[test] + fn stops_after_max_retries() { + let mut backoff = Backoff::default(); + let error = TransientAIError { message: String::new(), retry_after: None }; + for _ in 0..MAX_RETRIES { + let delay = backoff.next_delay(&error).expect("retry"); + assert!(delay <= MAX_BACKOFF); + } + assert_eq!(backoff.next_delay(&error), None); + } +} diff --git a/backend/windmill-ai/src/sse.rs b/backend/windmill-ai/src/sse.rs index ada50afca5..bdaa75025f 100644 --- a/backend/windmill-ai/src/sse.rs +++ b/backend/windmill-ai/src/sse.rs @@ -1,6 +1,6 @@ use std::collections::HashMap; -use eventsource_stream::Eventsource; +use eventsource_stream::{EventStreamError, Eventsource}; use indexmap::IndexMap; use reqwest::Response; use serde::Deserialize; @@ -8,12 +8,13 @@ use tokio_stream::StreamExt; use windmill_common::{error::Error, utils::rd_string}; use crate::{ - ai_google::{parse_gemini_sse_event, GeminiUsageMetadata}, + ai_google::{parse_gemini_sse_event, GeminiTurn, GeminiUsageMetadata}, ai_types::UrlCitation, ai_types::{ AnthropicExtraContent, ExtraContent, GoogleExtraContent, OpenAIFunction, OpenAIToolCall, }, query_builder::StreamEventSink, + retry::{transient_error, truncated_stream_error, StreamErrorBody}, types::{StreamingEvent, TokenUsage}, }; @@ -43,6 +44,8 @@ pub struct OpenAIChoiceDelta { #[derive(Deserialize)] pub struct OpenAIChoice { pub delta: Option, + #[serde(default)] + pub finish_reason: Option, } /// Nested prompt token details returned by the Chat Completions API. @@ -87,6 +90,8 @@ pub struct OpenAISSEEvent { pub choices: Option>, #[serde(default)] pub usage: Option, + #[serde(default)] + pub error: Option, } lazy_static::lazy_static! { @@ -99,6 +104,14 @@ lazy_static::lazy_static! { pub trait SSEParser { async fn parse_event_data(&mut self, data: &str) -> Result<(), Error>; + /// Whether the stream delivered the event that ends a complete response. + fn is_complete(&self) -> bool; + + /// Whether a stream that closed cleanly holds a complete response. + fn is_complete_at_close(&self) -> bool { + self.is_complete() + } + async fn parse_events(&mut self, response: Response) -> Result<(), Error> { let mut stream = response.bytes_stream().eventsource(); let mut consecutive_errors = 0; @@ -113,6 +126,18 @@ pub trait SSEParser { self.parse_event_data(&event.data).await?; } + // The body ends right after a transport error, so it cannot be skipped + // like a malformed event. A timeout spent the caller's whole budget. + Err(EventStreamError::Transport(_)) if self.is_complete() => break, + Err(EventStreamError::Transport(e)) => { + let message = + format!("The connection to the AI provider broke off mid-response: {e}"); + return Err(if e.is_timeout() { + Error::AIError(message) + } else { + transient_error(message, None) + }); + } Err(e) => { consecutive_errors += 1; tracing::error!("Failed to parse SSE event: {}", e); @@ -130,6 +155,9 @@ pub trait SSEParser { } } + if !self.is_complete_at_close() { + return Err(truncated_stream_error()); + } Ok(()) } } @@ -145,6 +173,9 @@ pub struct OpenAISSEParser { pub stream_event_processor: Box, /// Token usage from final chunk (when stream_options.include_usage is true) pub usage: Option, + /// Set by `[DONE]` or by a choice's `finish_reason`: OpenAI-compatible gateways do not + /// all send `[DONE]`. + complete: bool, } impl OpenAISSEParser { @@ -156,13 +187,19 @@ impl OpenAISSEParser { events_str: String::new(), stream_event_processor, usage: None, + complete: false, } } } impl SSEParser for OpenAISSEParser { + fn is_complete(&self) -> bool { + self.complete + } + async fn parse_event_data(&mut self, data: &str) -> Result<(), Error> { if data == "[DONE]" { + self.complete = true; return Ok(()); } @@ -173,13 +210,21 @@ impl SSEParser for OpenAISSEParser { .ok(); if let Some(event) = event { + if let Some(error) = event.error { + return Err(error.into_error("AI provider")); + } + // Extract usage from final chunk (when stream_options.include_usage is true) if let Some(usage) = event.usage { self.usage = Some(usage); } if let Some(mut choices) = event.choices.filter(|s| !s.is_empty()) { - if let Some(delta) = choices.remove(0).delta { + let choice = choices.remove(0); + if choice.finish_reason.is_some() { + self.complete = true; + } + if let Some(delta) = choice.delta { if let Some(reasoning) = delta.reasoning_content.filter(|s| !s.is_empty()) { self.accumulated_reasoning.push_str(&reasoning); let event = StreamingEvent::ReasoningTokenDelta { content: reasoning }; @@ -345,6 +390,8 @@ pub enum AnthropicSSEEvent { MessageDelta { #[serde(default)] usage: Option, + #[serde(default)] + delta: Option, }, #[serde(rename = "message_stop")] MessageStop {}, @@ -353,12 +400,18 @@ pub enum AnthropicSSEEvent { #[serde(rename = "error")] Error { #[serde(default)] - message: Option, + error: StreamErrorBody, }, #[serde(other)] Unknown, } +#[derive(Deserialize, Debug)] +pub struct AnthropicMessageDelta { + #[serde(default)] + pub stop_reason: Option, +} + /// Tracks state of a content block during streaming #[derive(Debug)] #[allow(dead_code)] @@ -391,6 +444,9 @@ pub struct AnthropicSSEParser { /// can be replayed before `tool_use` (required by Claude when thinking is on). pending_reasoning: Option, reasoning_attached: bool, + /// Set by `message_stop` or by the `stop_reason` of the `message_delta` sent just + /// before it, so a gateway that drops `message_stop` does not fail a whole answer. + complete: bool, } impl AnthropicSSEParser { @@ -407,11 +463,16 @@ impl AnthropicSSEParser { usage: None, pending_reasoning: None, reasoning_attached: false, + complete: false, } } } impl SSEParser for AnthropicSSEParser { + fn is_complete(&self) -> bool { + self.complete + } + async fn parse_event_data(&mut self, data: &str) -> Result<(), Error> { let event: Option = serde_json::from_str(data) .inspect_err(|e| { @@ -583,9 +644,8 @@ impl SSEParser for AnthropicSSEParser { .await?; } } - AnthropicSSEEvent::Error { message } => { - let error_msg = message.unwrap_or_else(|| "Unknown error".to_string()); - tracing::error!("Anthropic streaming error: {}", error_msg); + AnthropicSSEEvent::Error { error } => { + return Err(error.into_anthropic_error()); } AnthropicSSEEvent::MessageStart { message } => { // The only event carrying the prompt-side counts. `message_delta` @@ -595,7 +655,10 @@ impl SSEParser for AnthropicSSEParser { self.usage = Some(usage); } } - AnthropicSSEEvent::MessageDelta { usage } => { + AnthropicSSEEvent::MessageDelta { usage, delta } => { + if delta.is_some_and(|d| d.stop_reason.is_some()) { + self.complete = true; + } if let Some(u) = usage { match &mut self.usage { // Field by field, so the prompt counts from `message_start` @@ -614,10 +677,9 @@ impl SSEParser for AnthropicSSEParser { } } } + AnthropicSSEEvent::MessageStop {} => self.complete = true, // Ignore other events - AnthropicSSEEvent::MessageStop {} - | AnthropicSSEEvent::Ping {} - | AnthropicSSEEvent::Unknown => {} + AnthropicSSEEvent::Ping {} | AnthropicSSEEvent::Unknown => {} } } @@ -645,6 +707,7 @@ pub struct GeminiSSEParser { pub annotations: Vec, pub used_websearch: bool, pub usage: Option, + turn: GeminiTurn, } impl GeminiSSEParser { @@ -659,15 +722,25 @@ impl GeminiSSEParser { annotations: Vec::new(), used_websearch: false, usage: None, + turn: GeminiTurn::default(), } } } impl SSEParser for GeminiSSEParser { + fn is_complete(&self) -> bool { + self.turn.ended() + } + + fn is_complete_at_close(&self) -> bool { + self.turn.complete_at_close() + } + async fn parse_event_data(&mut self, data: &str) -> Result<(), Error> { let Some(parsed) = parse_gemini_sse_event(data)? else { return Ok(()); }; + self.turn.observe(&parsed); if let Some(reasoning) = parsed.reasoning.filter(|s| !s.is_empty()) { self.accumulated_reasoning.push_str(&reasoning); @@ -797,6 +870,8 @@ impl OpenAIResponsesUsage { pub struct OpenAIResponsesResponse { #[serde(default)] pub usage: Option, + #[serde(default)] + pub error: Option, } /// SSE event types for OpenAI Responses API streaming @@ -833,6 +908,20 @@ pub enum OpenAIResponsesSSEEvent { #[serde(rename = "response.incomplete")] Incomplete { response: OpenAIResponsesResponse }, + #[serde(rename = "response.failed")] + Failed { response: OpenAIResponsesResponse }, + + /// Flat (`code`, `message`) from OpenAI, nested under `error` from some compatible servers. + #[serde(rename = "error")] + Error { + #[serde(default)] + code: Option, + #[serde(default)] + message: Option, + #[serde(default)] + error: Option, + }, + /// Response created #[serde(rename = "response.created")] Created {}, @@ -890,6 +979,7 @@ pub struct OpenAIResponsesSSEParser { pub usage: Option, /// Reasoning summary parts seen so far, to separate them as paragraphs reasoning_summary_parts: usize, + response_ended: bool, } impl OpenAIResponsesSSEParser { @@ -906,11 +996,16 @@ impl OpenAIResponsesSSEParser { used_websearch: false, usage: None, reasoning_summary_parts: 0, + response_ended: false, } } } impl SSEParser for OpenAIResponsesSSEParser { + fn is_complete(&self) -> bool { + self.response_ended + } + async fn parse_event_data(&mut self, data: &str) -> Result<(), Error> { let event: Option = serde_json::from_str(data) .inspect_err(|e| { @@ -1010,11 +1105,22 @@ impl SSEParser for OpenAIResponsesSSEParser { OpenAIResponsesSSEEvent::Completed { response } | OpenAIResponsesSSEEvent::Incomplete { response } => { + self.response_ended = true; if let Some(usage) = response.usage { self.usage = Some(usage); } } + OpenAIResponsesSSEEvent::Failed { response } => { + return Err(response.error.unwrap_or_default().into_error("OpenAI")); + } + + OpenAIResponsesSSEEvent::Error { code, message, error } => { + return Err(error + .unwrap_or(StreamErrorBody { code, message, ..Default::default() }) + .into_error("OpenAI")); + } + OpenAIResponsesSSEEvent::ReasoningSummaryPartAdded {} => { self.reasoning_summary_parts += 1; if self.reasoning_summary_parts > 1 { @@ -1037,9 +1143,10 @@ impl SSEParser for OpenAIResponsesSSEParser { } } + OpenAIResponsesSSEEvent::Done {} => self.response_ended = true, + // Ignore other event types - OpenAIResponsesSSEEvent::Done {} - | OpenAIResponsesSSEEvent::Created {} + OpenAIResponsesSSEEvent::Created {} | OpenAIResponsesSSEEvent::InProgress {} | OpenAIResponsesSSEEvent::OutputItemDone {} | OpenAIResponsesSSEEvent::ContentPartAdded {} @@ -1093,6 +1200,117 @@ mod tests { } } + fn sse_response(chunks: Vec>) -> Response { + let body = reqwest::Body::wrap_stream(futures::stream::iter(chunks)); + Response::from(http::Response::new(body)) + } + + /// A stream that breaks off, or reports a failure inside its 200, must fail the call + /// rather than hand back what arrived as if it were the whole answer. + #[tokio::test] + async fn stream_failures_are_errors_not_partial_answers() { + use crate::retry::as_transient; + + const STARTED: &str = concat!( + "data: {\"type\":\"message_start\"}\n\n", + "data: {\"type\":\"content_block_start\",\"index\":0,\"content_block\":{\"type\":\"text\",\"text\":\"\"}}\n\n", + "data: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"text_delta\",\"text\":\"Hel\"}}\n\n", + ); + let anthropic = |chunks| async move { + AnthropicSSEParser::new(Box::new(NoopSink)) + .parse_events(sse_response(chunks)) + .await + }; + + let overloaded = anthropic(vec![ + Ok(STARTED), + Ok("data: {\"type\":\"error\",\"error\":{\"type\":\"overloaded_error\",\"message\":\"Overloaded\"}}\n\n"), + ]) + .await + .unwrap_err(); + assert!(as_transient(&overloaded).is_some(), "{overloaded}"); + + let refused = anthropic(vec![ + Ok(STARTED), + Ok("data: {\"type\":\"error\",\"error\":{\"type\":\"invalid_request_error\",\"message\":\"bad\"}}\n\n"), + ]) + .await + .unwrap_err(); + assert!(as_transient(&refused).is_none(), "{refused}"); + + let truncated = anthropic(vec![Ok(STARTED)]).await.unwrap_err(); + assert!(as_transient(&truncated).is_some(), "{truncated}"); + + let dropped = anthropic(vec![ + Ok(STARTED), + Err(std::io::Error::new( + std::io::ErrorKind::ConnectionReset, + "reset", + )), + ]) + .await + .unwrap_err(); + assert!(as_transient(&dropped).is_some(), "{dropped}"); + + anthropic(vec![ + Ok(STARTED), + Ok("data: {\"type\":\"message_stop\"}\n\n"), + ]) + .await + .unwrap(); + + anthropic(vec![ + Ok(STARTED), + Ok("data: {\"type\":\"message_delta\",\"delta\":{\"stop_reason\":\"end_turn\"}}\n\n"), + ]) + .await + .unwrap(); + + let truncated = OpenAISSEParser::new(Box::new(NoopSink)) + .parse_events(sse_response(vec![Ok( + "data: {\"choices\":[{\"delta\":{\"content\":\"Hi\"}}]}\n\n", + )])) + .await + .unwrap_err(); + assert!(as_transient(&truncated).is_some(), "{truncated}"); + + let quota = OpenAIResponsesSSEParser::new(Box::new(NoopSink)) + .parse_events(sse_response(vec![Ok( + "data: {\"type\":\"error\",\"error\":{\"type\":\"insufficient_quota\",\"code\":\"insufficient_quota\",\"message\":\"m\"}}\n\n", + )])) + .await + .unwrap_err(); + assert!(as_transient("a).is_none(), "{quota}"); + + GeminiSSEParser::new(Box::new(NoopSink)) + .parse_events(sse_response(vec![Ok( + "data: {\"candidates\":[{\"content\":{\"role\":\"model\",\"parts\":[{\"functionCall\":{\"name\":\"get_weather\",\"args\":{}}}]}}]}\n\n", + )])) + .await + .unwrap(); + + let gemini = |chunks| async move { + GeminiSSEParser::new(Box::new(NoopSink)) + .parse_events(sse_response(chunks)) + .await + }; + // Thinking spent the whole `maxOutputTokens`: a clean close with no event. + gemini(vec![]).await.unwrap(); + let dropped = gemini(vec![Err(std::io::Error::new( + std::io::ErrorKind::ConnectionReset, + "reset", + ))]) + .await + .unwrap_err(); + assert!(as_transient(&dropped).is_some(), "{dropped}"); + let truncated = gemini(vec![Ok( + "data: {\"candidates\":[{\"content\":{\"role\":\"model\",\"parts\":[{\"text\":\"Hi\"}]}}]}\n\n", + )]) + .await + .unwrap_err(); + assert!(as_transient(&truncated).is_some(), "{truncated}"); + } + /// The prompt-side counts arrive only on `message_start` and the completion total /// only on `message_delta`; a parser that keeps just the last one reports a request /// with no input tokens at all. diff --git a/backend/windmill-ai/src/types.rs b/backend/windmill-ai/src/types.rs index 7e2de09e01..59733e352c 100644 --- a/backend/windmill-ai/src/types.rs +++ b/backend/windmill-ai/src/types.rs @@ -1,3 +1,4 @@ +use indexmap::IndexMap; use serde::{Deserialize, Serialize}; use serde_json::value::RawValue; use std::collections::HashMap; @@ -269,6 +270,12 @@ pub struct ProviderResource { /// credentials exchange this crate does not perform. #[serde(default, deserialize_with = "empty_string_as_none")] pub token_url: Option, + /// The API token of a `cloudflare` resource, which names it `token` rather than `api_key`. + #[serde(default, deserialize_with = "empty_string_as_none")] + pub token: Option, + /// The account of a `cloudflare` resource, which its Workers AI URL is built from. + #[serde(default, deserialize_with = "empty_string_as_none")] + pub account_id: Option, } #[derive(Deserialize, Debug)] @@ -301,6 +308,11 @@ impl ProviderWithResource { } pub async fn get_base_url(&self, db: &DB) -> Result { + if self.kind == AIProvider::Cloudflare && self.resource.base_url.is_none() { + return crate::ai_providers::cloudflare_workers_ai_base_url( + self.resource.account_id.as_deref(), + ); + } self.kind .get_base_url(self.resource.base_url.clone(), db) .await @@ -320,7 +332,14 @@ impl ProviderWithResource { Ok(ProviderCredentials { provider: self.kind.clone(), base_url, - api_key: self.resource.api_key.clone(), + api_key: match self.kind { + AIProvider::Cloudflare => self + .resource + .api_key + .clone() + .or_else(|| self.resource.token.clone()), + _ => self.resource.api_key.clone(), + }, access_token: None, organization_id: None, user: None, @@ -515,7 +534,7 @@ pub struct OpenAPISchema { #[serde(skip_serializing_if = "Option::is_none")] pub items: Option>, #[serde(skip_serializing_if = "Option::is_none")] - pub properties: Option>>, + pub properties: Option>>, #[serde(skip_serializing_if = "Option::is_none")] pub required: Option>, #[serde(skip_serializing_if = "Option::is_none", rename = "oneOf")] @@ -542,9 +561,9 @@ pub struct OpenAPISchema { #[serde(skip_serializing_if = "Option::is_none", rename = "$ref")] pub ref_path: Option, #[serde(skip_serializing_if = "Option::is_none", rename = "$defs")] - pub defs: Option>>, + pub defs: Option>>, #[serde(skip_serializing_if = "Option::is_none")] - pub definitions: Option>>, + pub definitions: Option>>, // Schema composition #[serde(skip_serializing_if = "Option::is_none", rename = "allOf")] @@ -811,7 +830,7 @@ impl OpenAPISchema { // Merge properties (other's properties are added if key doesn't exist) if let Some(ref other_props) = other.properties { - let props = self.properties.get_or_insert_with(HashMap::new); + let props = self.properties.get_or_insert_with(IndexMap::new); for (key, value) in other_props { props.entry(key.clone()).or_insert_with(|| value.clone()); } @@ -862,13 +881,13 @@ impl OpenAPISchema { // Merge $defs and definitions if let Some(ref other_defs) = other.defs { - let defs = self.defs.get_or_insert_with(HashMap::new); + let defs = self.defs.get_or_insert_with(IndexMap::new); for (key, value) in other_defs { defs.entry(key.clone()).or_insert_with(|| value.clone()); } } if let Some(ref other_definitions) = other.definitions { - let definitions = self.definitions.get_or_insert_with(HashMap::new); + let definitions = self.definitions.get_or_insert_with(IndexMap::new); for (key, value) in other_definitions { definitions .entry(key.clone()) @@ -963,7 +982,29 @@ pub struct S3ObjectWithType { #[cfg(test)] mod tests { use super::*; - use std::collections::HashMap; + + /// The schema is part of the prompt prefix the provider caches on, so its keys must + /// serialize in the same order on every request. + #[test] + fn strict_schema_keeps_the_defined_property_order() { + let keys = [ + "zeta", "alpha", "mid", "beta", "omega", "gamma", "delta", "kappa", + ]; + let props = keys + .iter() + .map(|k| format!(r#""{k}":{{"type":"string"}}"#)) + .collect::>() + .join(","); + let mut schema: OpenAPISchema = + serde_json::from_str(&format!(r#"{{"type":"object","properties":{{{props}}}}}"#)) + .unwrap(); + schema.make_strict(); + + let value = serde_json::to_value(&schema).unwrap(); + let serialized: Vec<&String> = value["properties"].as_object().unwrap().keys().collect(); + assert_eq!(serialized, keys); + assert_eq!(value["required"], serde_json::json!(keys)); + } /// Whichever provider served the request, `input_tokens` has to end up meaning the /// whole prompt. Leaving the Anthropic shape as reported under-states it by the @@ -1230,7 +1271,7 @@ mod tests { #[test] fn test_make_strict_defs() { let def_schema = object_schema(vec![("prop", string_schema())]); - let mut defs = HashMap::new(); + let mut defs = IndexMap::new(); defs.insert("MyType".to_string(), Box::new(def_schema)); let mut schema = OpenAPISchema { defs: Some(defs), ..Default::default() }; @@ -1256,7 +1297,7 @@ mod tests { #[test] fn test_make_strict_definitions() { let def_schema = object_schema(vec![("prop", string_schema())]); - let mut definitions = HashMap::new(); + let mut definitions = IndexMap::new(); definitions.insert("MyType".to_string(), Box::new(def_schema)); let mut schema = OpenAPISchema { definitions: Some(definitions), ..Default::default() }; @@ -1510,7 +1551,7 @@ mod tests { #[test] fn test_flatten_all_of_preserves_defs() { let def_schema = object_schema(vec![("field", string_schema())]); - let mut defs = HashMap::new(); + let mut defs = IndexMap::new(); defs.insert("MyType".to_string(), Box::new(def_schema)); let schema_with_defs = OpenAPISchema { defs: Some(defs), ..Default::default() }; @@ -1624,7 +1665,7 @@ mod tests { schema_url: Some("http://json-schema.org/draft-07/schema#".to_string()), ..Default::default() }; - let mut defs = HashMap::new(); + let mut defs = IndexMap::new(); defs.insert("MyType".to_string(), Box::new(def_schema)); let mut schema = OpenAPISchema { diff --git a/backend/windmill-api-auth/src/auth.rs b/backend/windmill-api-auth/src/auth.rs index 869f40e14c..720111e89d 100644 --- a/backend/windmill-api-auth/src/auth.rs +++ b/backend/windmill-api-auth/src/auth.rs @@ -104,6 +104,32 @@ pub struct ExpiringAuthCache { pub job_id: Option, } +/// Whether `target` belongs to the flow run of the job `job_id`: the job itself, one of its +/// ancestors, or a job whose parent or root is one of them (a sibling step, a loop or branch +/// iteration, any job pushed as a child of one of them). A lookup that fails reads as no. +pub async fn job_in_same_flow_run(db: &DB, job_id: uuid::Uuid, target: uuid::Uuid) -> bool { + if job_id == target { + return true; + } + sqlx::query_scalar!( + "SELECT EXISTS ( + SELECT 1 FROM v2_job o, v2_job t, + LATERAL (SELECT ARRAY_REMOVE(ARRAY[o.id, o.parent_job, o.root_job, + o.flow_innermost_root_job], NULL) AS run) l + WHERE o.id = $1 AND t.id = $2 AND t.workspace_id = o.workspace_id + AND (t.id = ANY(l.run) OR t.parent_job = ANY(l.run) + OR t.root_job = ANY(l.run) OR t.flow_innermost_root_job = ANY(l.run)) + )", + job_id, + target + ) + .fetch_one(db) + .await + .ok() + .flatten() + .unwrap_or(false) +} + pub struct AuthCache { db: DB, superadmin_secret: Option, @@ -1131,8 +1157,20 @@ pub async fn resolve_opt_job_authed( return Err((err, parts)); } } + let own_job_runtime_route = match opt_job_authed.job_id { + Some(job_id) if opt_job_authed.authed.scopes.is_some() => { + crate::scopes::is_own_job_runtime_route(path, method, job_id) + || match crate::scopes::flow_run_read_route_job(path, method) { + Some(target) => { + job_in_same_flow_run(&cache.db, job_id, target).await + } + None => false, + } + } + _ => false, + }; let authed = &mut opt_job_authed.authed; - if authed.scopes.is_some() { + if authed.scopes.is_some() && !own_job_runtime_route { transform_old_scope_to_new_scope(authed.scopes.as_mut()); if let Err(err) = crate::scopes::check_scopes_for_route( diff --git a/backend/windmill-api-auth/src/lib.rs b/backend/windmill-api-auth/src/lib.rs index 12ad5f2654..30c3bb65af 100644 --- a/backend/windmill-api-auth/src/lib.rs +++ b/backend/windmill-api-auth/src/lib.rs @@ -35,6 +35,7 @@ use windmill_common::{ }; use scopes::ScopeDefinition; +use windmill_common::scopes::scope_contains; // Re-export key auth types and functions pub use auth::{ @@ -475,11 +476,11 @@ pub fn is_effectively_unscoped(scopes: Option<&[String]>) -> bool { /// refused. An admin check is not a substitute — it answers for the user behind the /// token, not for the token's own scopes. /// -/// This bounds the token making the request, not every route to the key. A job token -/// is minted unscoped from its owner's privileges, so a `jobs:run` token still reaches -/// the key indirectly by running a job as a workspace admin — the same property that -/// lets git-sync export it. Confining that means not inheriting unscoped privilege -/// into job tokens, which is a far wider change than this guard. +/// This bounds the token making the request, not every route to the key. The jobs a +/// scoped user token runs get a token with their owner's privileges, capped only by the +/// runnable's own `job_token_scopes` (see [`caller_scope_ceiling`]), so a `jobs:run` +/// token still reaches the key indirectly by running an unrestricted job as a workspace +/// admin — the same property that lets git-sync export it. pub fn forbid_scoped_token_workspace_key(authed: &ApiAuthed) -> error::Result<()> { if is_effectively_unscoped(authed.scopes.as_deref()) { return Ok(()); @@ -492,6 +493,50 @@ pub fn forbid_scoped_token_workspace_key(authed: &ApiAuthed) -> error::Result<() )) } +/// For a route that authenticates its token itself rather than through the route layer +/// (which checks scopes): a job token restricted by `job_token_scopes` must still hold +/// `required` there. Other credentials keep the access those routes always gave them. +pub fn check_job_token_scope(authed: &ApiAuthed, required: F) -> error::Result<()> +where + F: FnOnce() -> String, +{ + if authed.job_id.is_some() { + check_scopes(authed, required) + } else { + Ok(()) + } +} + +/// The cap on the token of a job pushed on `authed`'s request, to pass to `push` as its +/// scope ceiling. A job token caps every job it starts at its own job's scopes, so a +/// restricted job cannot widen its reach by running another one; the job row is the +/// source rather than the token's claims, which the MCP proxy narrows per route. Any +/// other credential caps nothing: the jobs it runs act as their owner, as they always did. +pub async fn caller_scope_ceiling( + db: &windmill_common::DB, + authed: &ApiAuthed, +) -> error::Result>> { + let Some(job_id) = authed.job_id else { + return Ok(None); + }; + // A job token is minted with scopes exactly when its job is restricted, so an unscoped one + // has nothing to read. + if authed.scopes.is_none() { + return Ok(None); + } + let row = sqlx::query_scalar!( + "SELECT job_token_scopes FROM job_perms WHERE job_id = $1", + job_id + ) + .fetch_optional(db) + .await?; + Ok(match row { + Some(scopes) => scopes, + // The job left the queue and its row was swept: only the token's own claims are left. + None => authed.scopes.clone(), + }) +} + /// Enforce monotonic privilege when a token lifecycle endpoint mints or rescopes /// a credential on behalf of `authed`: the resulting credential must never be /// more privileged than the caller's own token. @@ -604,83 +649,6 @@ fn first_filter_tags(scopes: Option<&[String]>) -> Option> { }) } -/// Whether `caller` grants at least everything `requested` grants (directional -/// containment). -/// -/// This is intentionally NOT `ScopeDefinition::includes`: that method answers -/// "does this scope grant access to a required action" using OR semantics over -/// resources (any overlap counts, and a `*` on either side matches), which is -/// correct for access checks but unsafe for subset checks — it would let a -/// token scoped to `scripts:read:f/team/a` mint `scripts:read:*` or -/// `scripts:read:f/team/a,f/other/b`. Subset containment instead requires that -/// EVERY requested resource is covered by SOME caller resource. -fn scope_contains(caller: &ScopeDefinition, requested: &ScopeDefinition) -> bool { - if caller.domain != requested.domain { - return false; - } - - // write subsumes read; otherwise the action must match exactly. - match (caller.action.as_str(), requested.action.as_str()) { - (c, r) if c == r || (c == "write" && r == "read") => {} - // Apps only: `write` covers `run` (see `ScopeDefinition::includes`), so an - // app-editor token can mint the narrower run-only credential. - ("write", "run") if caller.domain == "apps" => {} - _ => return false, - } - - if caller.domain == "jobs" && caller.action == "run" { - match (&caller.kind, &requested.kind) { - (Some(caller_kind), Some(requested_kind)) if caller_kind != requested_kind => { - return false - } - // Caller pinned to a kind, but the request covers any kind. - (Some(_), None) => return false, - _ => {} - } - } - - match (&caller.resource, &requested.resource) { - // Caller is unrestricted on resources: covers everything. - (None, _) => true, - // Caller is resource-restricted but the request is not: broader, unless the - // caller lists `*` and so already spans every path. Kept in step with - // `ScopeDefinition::includes`, which accepts that same grant for a - // whole-collection read: what a token may exercise, it may also delegate. - (Some(caller_resources), None) => caller_resources.iter().any(|r| r == "*"), - (Some(caller_resources), Some(requested_resources)) => { - resource_set_contains(caller_resources, requested_resources) - } - } -} - -/// Every resource in `requested` must be covered by some resource in `caller`. -fn resource_set_contains(caller: &[String], requested: &[String]) -> bool { - if caller.iter().any(|r| r == "*") { - return true; - } - requested - .iter() - .all(|req| req != "*" && caller.iter().any(|c| resource_covers(c, req))) -} - -/// Directional: does the single caller resource pattern cover `requested`? -/// `caller` may be an exact path or a `/*` subtree wildcard; `requested` -/// may itself be a subtree wildcard, in which case the whole requested subtree -/// must fall within the caller's subtree. -fn resource_covers(caller: &str, requested: &str) -> bool { - if caller == requested { - return true; - } - let Some(prefix) = caller.strip_suffix("/*") else { - // An exact caller resource only covers itself (handled above). - return false; - }; - let requested_base = requested.strip_suffix("/*").unwrap_or(requested); - requested_base == prefix - || (requested_base.starts_with(prefix) - && requested_base.as_bytes().get(prefix.len()) == Some(&b'/')) -} - /// Returns a predicate that checks whether `path` is within the token's /// scope for `{domain}:{action}:{path}`. For tokens without scope /// restrictions (no scopes at all, or only `if_jobs:filter_tags:*` scopes), diff --git a/backend/windmill-api-auth/src/scopes.rs b/backend/windmill-api-auth/src/scopes.rs index e100cf88c6..21f916bf94 100644 --- a/backend/windmill-api-auth/src/scopes.rs +++ b/backend/windmill-api-auth/src/scopes.rs @@ -6,464 +6,9 @@ * LICENSE-AGPL for a copy of the license. */ -use itertools::Itertools; -use serde::{Deserialize, Serialize}; -use std::collections::HashSet; use windmill_common::error::{Error, Result}; -/// Comprehensive scope system for JWT token authorization -/// -/// Scopes follow the format: {domain}:{action}[:{resource}] -/// Examples: -/// - "jobs:read" - Read access to jobs -/// - "scripts:write:f/folder/*" - Write access to scripts in a folder -/// - "*" - Full access (superuser) - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct ScopeDefinition { - pub domain: String, - pub action: String, - pub kind: Option, // For jobs:run:kind (optional) - pub resource: Option>, -} - -impl ScopeDefinition { - pub fn new( - domain: &str, - action: &str, - kind: Option<&str>, - resource: Option>, - ) -> Self { - Self { - domain: domain.to_string(), - action: action.to_string(), - kind: kind.map(|s| s.to_string()), - resource: resource, - } - } - - pub fn from_scope_string(scope: &str) -> Result { - let parts: Vec<&str> = scope.split(':').collect(); - - let into_owned_vec = |resources: &str| -> Vec { - let resources = resources - .split(",") - .collect_vec() - .into_iter() - .map(ToOwned::to_owned) - .collect_vec(); - - resources - }; - - match parts.len() { - 2 => Ok(Self::new(parts[0], parts[1], None, None)), // domain:action - 3 => { - if parts[0] == "jobs" && parts[1] == "run" { - Ok(Self::new(parts[0], parts[1], Some(parts[2]), None)) - } else { - Ok(Self::new( - parts[0], - parts[1], - None, - Some(into_owned_vec(parts[2])), - )) - } - } - 4 => { - if parts[0] == "jobs" && parts[1] == "run" { - Ok(Self::new( - parts[0], - parts[1], - Some(parts[2]), - Some(into_owned_vec(parts[3])), - )) - } else { - Err(Error::BadRequest(format!( - "Invalid 4-part scope: {}", - scope - ))) - } - } - _ => Err(Error::BadRequest(format!( - "Invalid scope format: {}", - scope - ))), - } - } - - pub fn as_string(&self) -> String { - match (&self.kind, &self.resource) { - (Some(kind), Some(resource)) => { - format!( - "{}:{}:{}:{}", - self.domain, - self.action, - kind, - resource.join(",") - ) - } - (Some(kind), None) => { - format!("{}:{}:{}", self.domain, self.action, kind) - } - (None, Some(resource)) => { - format!("{}:{}:{}", self.domain, self.action, resource.join(",")) - } - (None, None) => format!("{}:{}", self.domain, self.action), - } - } - - pub fn includes(&self, other: &ScopeDefinition) -> bool { - if self.domain != other.domain { - return false; - } - - match (self.action.as_str(), other.action.as_str()) { - (a, b) if (a == "write" && b == "read") || (a == b) => {} - // Apps only: `write` can rewrite the app and its policy, so it also covers - // running its components. Not general — `jobs:write` must not grant - // `jobs:run`. The resource check below still confines it to the same app. - ("write", "run") if self.domain == "apps" => {} - _ => return false, - } - - if self.domain == "jobs" && self.action == "run" { - match (&self.kind, &other.kind) { - (Some(self_kind), Some(other_kind)) => { - if self_kind != other_kind { - return false; - } - } - (Some(_), None) => { - return false; - } - (None, _) => { - return true; - } - } - } - - match (&self.resource, &other.resource) { - (Some(self_resources), Some(other_resources)) => { - resources_match(self_resources, other_resources) - } - // A requirement naming no path is the whole domain, so only a grant that - // itself spans every path satisfies it. `*` is that grant — the scope UI - // accepts it as a resource path and `resources_match` already reads it as - // everything — while any listed path leaves the collection unauthorized. - (Some(self_resources), None) => self_resources.iter().any(|r| r == "*"), - (None, _) => true, - } - } -} - -fn resources_match(scope_resources: &[String], accepted_resources: &[String]) -> bool { - if scope_resources.contains(&"*".to_string()) || accepted_resources.contains(&"*".to_string()) { - return true; - } - - if scope_resources.len() <= 4 && accepted_resources.len() <= 4 { - return resources_match_small(scope_resources, accepted_resources); - } - - resources_match_large(scope_resources, accepted_resources) -} - -fn resources_match_small(scope_resources: &[String], accepted_resources: &[String]) -> bool { - for required in accepted_resources { - for scope_resource in scope_resources { - if resource_matches_pattern(scope_resource, required) { - return true; - } - } - } - false -} - -fn resources_match_large(scope_resources: &[String], accepted_resources: &[String]) -> bool { - let mut exact_matches = HashSet::new(); - let mut patterns = Vec::new(); - - for scope_resource in scope_resources { - if scope_resource.contains('*') { - patterns.push(scope_resource); - } else { - exact_matches.insert(scope_resource); - } - } - - for accepted_resource in accepted_resources { - if exact_matches.contains(accepted_resource) { - return true; - } - - for pattern in &patterns { - if resource_matches_pattern(pattern, accepted_resource) { - return true; - } - } - } - - false -} - -fn resource_matches_pattern(scope_resource: &str, accepted_resource: &str) -> bool { - if scope_resource == accepted_resource { - return true; - } - - let matches_wildcard = |pattern: &str, resource: &str| -> bool { - if !pattern.ends_with("/*") { - return false; - } - - let prefix = &pattern[..pattern.len() - 2]; - - if !resource.starts_with(prefix) { - return false; - } - - // If the resource is exactly the prefix, it matches - if resource.len() == prefix.len() { - return true; - } - - // If the resource is longer, the next character must be '/' for a valid match - // This prevents "u/user" from matching "u/use/*" - resource.chars().nth(prefix.len()) == Some('/') - }; - - // Check if either resource is a wildcard pattern and matches the other - matches_wildcard(scope_resource, accepted_resource) - || matches_wildcard(accepted_resource, scope_resource) -} - -// ───────────────────────────────────────────────────────────────── -// Route-level scope checking -// ───────────────────────────────────────────────────────────────── - -/// Available scope domains (top-level API categories) -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -pub enum ScopeDomain { - // Core resource domains - Jobs, - Scripts, - /// The `/data_metrics` catalog. Its own domain, NOT an alias of `Scripts`: a - /// `data_metrics:read` token must reach only this route, never the broader - /// `/scripts` routes (some of which do no further scope check). - DataMetrics, - Flows, - FlowConversations, - Apps, - Variables, - Resources, - Schedules, - Folders, - Users, - Groups, - Workspaces, - - // Trigger domains - HttpTriggers, - WebsocketTriggers, - KafkaTriggers, - NatsTriggers, - MqttTriggers, - AmqpTriggers, - SqsTriggers, - GcpTriggers, - AzureTriggers, - PostgresTriggers, - EmailTriggers, - - // Native trigger domains - NativeTriggers, - TriggersHistory, - - // System domains - Audit, - Settings, - Workers, - ServiceLogs, - Configs, - OAuth, - AI, - AiEvals, // AI agent eval datasets - - Indexer, - Teams, // Microsoft Teams integration - GitSync, // Git synchronization - - // Special domains - Capture, // Webhook capture - Drafts, // Draft resources - Favorites, // User favorites - Inputs, // Input templates - JobHelpers, // Job helper functions - ConcurrencyGroups, // Concurrency groups - Oidc, // OpenID Connect - Openapi, // OpenAPI generation - - // Additional domains - Acls, // Granular access control lists - RawApps, // Raw application data - AgentWorkers, // Agent workers management - Mcp, // MCP - Docs, // Self-hosted documentation search (read-only) -} - -impl ScopeDomain { - pub fn as_str(&self) -> &'static str { - match self { - Self::Jobs => "jobs", - Self::Scripts => "scripts", - Self::DataMetrics => "data_metrics", - Self::Flows => "flows", - Self::FlowConversations => "flow_conversations", - Self::Apps => "apps", - Self::Variables => "variables", - Self::Resources => "resources", - Self::Schedules => "schedules", - Self::Folders => "folders", - Self::Users => "users", - Self::Groups => "groups", - Self::Workspaces => "workspaces", - Self::HttpTriggers => "http_triggers", - Self::WebsocketTriggers => "websocket_triggers", - Self::KafkaTriggers => "kafka_triggers", - Self::NatsTriggers => "nats_triggers", - Self::MqttTriggers => "mqtt_triggers", - Self::AmqpTriggers => "amqp_triggers", - Self::SqsTriggers => "sqs_triggers", - Self::GcpTriggers => "gcp_triggers", - Self::AzureTriggers => "azure_triggers", - Self::PostgresTriggers => "postgres_triggers", - Self::EmailTriggers => "email_triggers", - Self::NativeTriggers => "native_triggers", - Self::TriggersHistory => "triggers_history", - Self::Audit => "audit", - Self::Settings => "settings", - Self::Workers => "workers", - Self::ServiceLogs => "service_logs", - Self::Configs => "configs", - Self::OAuth => "oauth", - Self::AI => "ai", - Self::AiEvals => "ai_evals", - Self::Capture => "capture", - Self::Drafts => "drafts", - Self::Favorites => "favorites", - Self::Inputs => "inputs", - Self::JobHelpers => "job_helpers", - Self::ConcurrencyGroups => "concurrency_groups", - Self::Oidc => "oidc", - Self::Openapi => "openapi", - Self::Acls => "acls", - Self::RawApps => "raw_apps", - Self::AgentWorkers => "agent_workers", - Self::Indexer => "indexer", - Self::Teams => "teams", - Self::GitSync => "git_sync", - Self::Mcp => "mcp", - Self::Docs => "docs", - } - } - - pub fn from_str(s: &str) -> Option { - match s { - "jobs" | "jobs_u" => Some(Self::Jobs), - "scripts" => Some(Self::Scripts), - // A distinct domain, not an alias of `scripts` (see the enum variant): - // a `data_metrics:read` token must not reach the broader /scripts routes. - "data_metrics" => Some(Self::DataMetrics), - "flows" => Some(Self::Flows), - "flow_conversations" => Some(Self::FlowConversations), - "apps" | "apps_u" => Some(Self::Apps), - "variables" => Some(Self::Variables), - "resources" => Some(Self::Resources), - "schedules" => Some(Self::Schedules), - "folders" => Some(Self::Folders), - "users" => Some(Self::Users), - "groups" => Some(Self::Groups), - "workspaces" => Some(Self::Workspaces), - "http_triggers" => Some(Self::HttpTriggers), - "websocket_triggers" => Some(Self::WebsocketTriggers), - "kafka_triggers" => Some(Self::KafkaTriggers), - "nats_triggers" => Some(Self::NatsTriggers), - "mqtt_triggers" => Some(Self::MqttTriggers), - "amqp_triggers" => Some(Self::AmqpTriggers), - "sqs_triggers" => Some(Self::SqsTriggers), - "gcp_triggers" => Some(Self::GcpTriggers), - "azure_triggers" => Some(Self::AzureTriggers), - "postgres_triggers" => Some(Self::PostgresTriggers), - "email_triggers" => Some(Self::EmailTriggers), - "audit" => Some(Self::Audit), - "settings" => Some(Self::Settings), - "workers" => Some(Self::Workers), - "service_logs" => Some(Self::ServiceLogs), - "configs" => Some(Self::Configs), - "oauth" => Some(Self::OAuth), - "ai" => Some(Self::AI), - "ai_evals" => Some(Self::AiEvals), - "indexer" | "srch" => Some(Self::Indexer), - "teams" => Some(Self::Teams), - "native_triggers" => Some(Self::NativeTriggers), - "triggers_history" => Some(Self::TriggersHistory), - "git_sync" | "github_app" => Some(Self::GitSync), - "capture" => Some(Self::Capture), - "drafts" => Some(Self::Drafts), - "favorites" => Some(Self::Favorites), - "inputs" => Some(Self::Inputs), - "job_helpers" => Some(Self::JobHelpers), - "concurrency_groups" => Some(Self::ConcurrencyGroups), - "oidc" => Some(Self::Oidc), - "openapi" => Some(Self::Openapi), - "acls" => Some(Self::Acls), - "raw_apps" => Some(Self::RawApps), - "agent_workers" => Some(Self::AgentWorkers), - "mcp" => Some(Self::Mcp), - "docs" => Some(Self::Docs), - _ => None, - } - } -} - -/// Available scope actions -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -pub enum ScopeAction { - Read, // GET operations, list, view - Write, // POST, PUT, PATCH, DELETE operations, create, update, delete - Run, // Special action for running (scripts, flows, etc.) -} - -impl ScopeAction { - pub fn as_str(&self) -> &'static str { - match self { - Self::Read => "read", - Self::Write => "write", - Self::Run => "run", - } - } - - pub fn from_str(s: &str) -> Option { - match s { - "read" => Some(Self::Read), - "write" => Some(Self::Write), - "delete" => Some(Self::Write), - "run" => Some(Self::Run), - _ => None, - } - } - - /// Check if this action includes another action - /// Write includes Read - pub fn includes(&self, other: &ScopeAction) -> bool { - match (self, other) { - (ScopeAction::Write, ScopeAction::Read) => true, - (ScopeAction::Run, ScopeAction::Read) => true, - (a, b) => a == b, - } - } -} +pub use windmill_common::scopes::{ScopeAction, ScopeDefinition, ScopeDomain}; pub fn check_route_access( token_scopes: &[String], @@ -625,10 +170,16 @@ const FLOW_JOBS: [&'static str; 6] = [ /// scoped by the agent's path like a deployed runnable's, as `jobs:run:agents:`. const AGENT_JOBS: [&'static str; 1] = ["jobs/run/agent"]; +/// Lock resolution for scripts, apps and flows, scoped as `jobs:run:dependencies:`. +/// Resolving a package can run its build or install scripts on the worker, so this is +/// still code execution: the scope only takes away the direct ways to run code. +const DEPENDENCY_JOBS: [&'static str; 2] = ["jobs/run/dependencies", "jobs/run/flow_dependencies"]; + lazy_static::lazy_static! { static ref RUN_PATH_ACTIONS: Vec<&'static str> = { - let mut v = vec!["jobs/resume/", "jobs/run/batch_rerun_jobs", "jobs/run/workflow_as_code", "jobs/run/dependencies","jobs/run/flow_dependencies", "apps_u/execute_component", "apps_u/upload_s3_file"]; + let mut v = vec!["jobs/resume/", "jobs/run/batch_rerun_jobs", "jobs/run/workflow_as_code", "apps_u/execute_component", "apps_u/upload_s3_file"]; + v.extend(DEPENDENCY_JOBS); v.extend(SCRIPT_JOBS); v.extend(FLOW_JOBS); v.extend(AGENT_JOBS); @@ -636,7 +187,38 @@ lazy_static::lazy_static! { }; } +/// The job routes a `jobs:cancel` scope reaches, as workspaced route suffixes. A +/// path-scoped `jobs:cancel:` is resource-blind here like every scope; the +/// handlers confine it through `job_cancel_path_confinement`, so a cancel route added +/// here without that check would serve a path-scoped token every job. +const CANCEL_PATH_ACTIONS: [&'static str; 4] = [ + "jobs_u/queue/cancel/", + "jobs_u/queue/force_cancel/", + "jobs_u/queue/cancel_persistent/", + "jobs/queue/cancel_selection", +]; + +fn is_cancel_route(method: &str, route_path: &str) -> bool { + if !method.eq_ignore_ascii_case("POST") { + return false; + } + let mut parts = route_path.splitn(5, '/'); + let (Some(""), Some("api"), Some("w"), Some(_), Some(suffix)) = ( + parts.next(), + parts.next(), + parts.next(), + parts.next(), + parts.next(), + ) else { + return false; + }; + CANCEL_PATH_ACTIONS.iter().any(|p| suffix.starts_with(p)) +} + fn map_http_method_to_action(method: &str, route_path: &str) -> ScopeAction { + if is_cancel_route(method, route_path) { + return ScopeAction::Cancel; + } if RUN_PATH_ACTIONS .iter() .any(|run_path| route_path.contains(run_path)) @@ -651,11 +233,13 @@ fn map_http_method_to_action(method: &str, route_path: &str) -> ScopeAction { } } -/// Checks the route path to determine the runnable kind ("agents", "flows" or "scripts"). +/// Checks the route path to determine the runnable kind ("dependencies", "agents", "flows" +/// or "scripts"). /// /// The order of checks is important: -/// - Flow-related paths are checked first to avoid false positives, as some flow paths -/// (e.g., `/run_preview_flow`) share prefixes with script paths (e.g., `/run_preview`). +/// - Dependency paths are checked before flow paths: `flow_dependencies` starts with `f`. +/// - Flow-related paths are checked before script paths to avoid false positives, as some +/// flow paths (e.g., `/run_preview_flow`) share prefixes with script paths (e.g., `/run_preview`). /// /// Returns the kind based on the match, or `None` if no match is found. fn determine_kind_from_route(route_path: &str) -> Option { @@ -671,6 +255,12 @@ fn determine_kind_from_route(route_path: &str) -> Option { { return None; } + if DEPENDENCY_JOBS + .iter() + .any(|path| route_path.starts_with(path)) + { + return Some("dependencies".to_string()); + } if AGENT_JOBS.iter().any(|path| route_path.starts_with(path)) { return Some("agents".to_string()); } @@ -726,7 +316,7 @@ fn extract_domain_from_route( /// Every entry is keyed by a job id and confines an authenticated caller to its own /// runnable — through `require_job_read_access`, through its own `jobs:run:flows:` /// check, or, where an approval token or resume secret bypasses that gate, through a -/// direct `require_job_within_run_scope`. The one exception is +/// direct `require_job_within_read_scope`. The one exception is /// `jobs_u/get_root_job_id/`, which has no check at all but discloses only flow lineage, /// to anyone, authenticated or not. Workspace-wide enumeration (`jobs/list`, counts, /// exports) and credential minting (`job_view_token`) are deliberately absent — those are @@ -943,23 +533,169 @@ fn resource_metadata_route_allowed(suffix: &str) -> bool { || suffix.starts_with("resources/type/") } -/// The `jobs:run` scopes a token's job reads are confined to, or `None` when they are -/// not confined to particular runnables. +/// The by-id and list routes a path-scoped `jobs:read:` reaches, as workspaced +/// route suffixes. Every other read in the domain is refused to it: counts, exports, +/// uuid listings, queue positions and dispatch edges aggregate over the workspace, and +/// the view-token and resume-url routes mint a credential (`approval_info` withholds +/// its view token from a confined caller for the same reason). +/// +/// Each entry confines the caller in its handler, through `require_job_read_access` or +/// a direct `require_job_within_read_scope`. `jobs/result_by_id/` is absent because it +/// resolves a node up through the enclosing flows, past the job the check admitted. +/// A route added here without that check serves a path-scoped token every job. +const READ_PATH_SCOPED_BY_ID_PATHS: [&'static str; 25] = [ + "jobs_u/get/", + "jobs_u/get_logs/", + "jobs_u/get_completed_logs_tail/", + "jobs_u/get_flow_all_logs/", + "jobs_u/get_flow_all_logs_structured/", + "jobs_u/get_flow_all_results/", + "jobs_u/get_args/", + "jobs_u/get_flow_debug_info/", + "jobs_u/completed/get/", + "jobs_u/completed/get_result/", + "jobs_u/completed/get_result_maybe/", + "jobs_u/completed/get_timing/", + "jobs_u/dispatch_events/", + "jobs_u/getupdate/", + "jobs_u/getupdate_sse/", + "jobs_u/get_log_file/", + "jobs_u/get_flow/", + "jobs_u/flow/approval_info/", + "jobs/run_progress/", + "jobs/flow/user_states/", + "jobs/completed/get/", + "jobs/completed/get_result/", + "jobs/completed/get_result_maybe/", + "jobs/completed/get_timing/", + "jobs/get_otel_traces/", +]; + +/// Matched whole: `jobs/list` is also a prefix of the uuid listings. Their handlers +/// filter in SQL to [`job_list_path_filter`]. +const READ_PATH_SCOPED_LIST_PATHS: [&'static str; 3] = + ["jobs/list", "jobs/completed/list", "jobs/queue/list"]; + +fn is_read_path_scoped_route(suffix: &str) -> bool { + READ_PATH_SCOPED_LIST_PATHS.contains(&suffix) + || READ_PATH_SCOPED_BY_ID_PATHS + .iter() + .any(|p| suffix.starts_with(p)) +} + +/// The paths of a `jobs:read:` scope, or `None` when it spans every job: no +/// path part, or `*` among them. +fn job_read_scope_paths(scope: &ScopeDefinition) -> Option<&[String]> { + scope + .resource + .as_deref() + .filter(|paths| !paths.iter().any(|p| p == "*")) +} + +/// The runnables a token's job reads are confined to: those its `jobs:run` scopes may +/// start, and those its `jobs:read:` scopes name. +pub struct JobReadConfinement { + run: Vec, + read_paths: crate::ScopePathFilter, +} + +impl JobReadConfinement { + /// Whether a job that ran `runnable_path` as `kind` (`scripts`, `flows`, `agents` or + /// `dependencies`) is inside the confinement. A read path admits every kind; a run + /// scope only its own. + pub fn admits(&self, kind: &str, runnable_path: &str) -> bool { + if self.read_paths.allows(runnable_path) { + return true; + } + // Lock jobs and previews store the path their caller sent. The matcher reads `*` + // in it as a pattern, and `,`/`:` are scope syntax: such a path needs a grant + // covering every path, as enqueuing it did. + let resource = windmill_common::auth::is_scope_literal_path(runnable_path) + .then(|| vec![runnable_path.to_string()]); + let required = ScopeDefinition::new( + ScopeDomain::Jobs.as_str(), + ScopeAction::Run.as_str(), + Some(kind), + resource, + ); + self.run.iter().any(|scope| scope.includes(&required)) + } + + /// Whether a `jobs:read:` scope is part of the confinement. + pub fn has_read_paths(&self) -> bool { + matches!(&self.read_paths, crate::ScopePathFilter::Restricted { exact, prefix } + if !exact.is_empty() || !prefix.is_empty()) + } +} + +fn push_read_paths(paths: &[String], exact: &mut Vec, prefix: &mut Vec) { + for path in paths { + match path.strip_suffix("/*") { + Some(p) => prefix.push(p.to_string()), + None => exact.push(path.clone()), + } + } +} + +/// The paths a token's job lists are filtered to. Decided by its read grants alone, +/// unlike [`job_read_confinement`]: a `jobs:run` scope grants no enumeration, so even a +/// bare one beside a `jobs:read:` leaves the lists on those paths. Unrestricted +/// for an unscoped token, a `jobs:read` naming no path (or `*`), and `jobs:write`. +pub fn job_list_path_filter(scopes: Option<&[String]>) -> crate::ScopePathFilter { + let mut exact = Vec::new(); + let mut prefix = Vec::new(); + let mut scoped = false; + for scope in scopes + .unwrap_or_default() + .iter() + .filter(|s| !s.starts_with("if_jobs:filter_tags:")) + { + scoped = true; + let Ok(scope) = ScopeDefinition::from_scope_string(scope) else { + continue; + }; + if ScopeDomain::from_str(&scope.domain) != Some(ScopeDomain::Jobs) { + continue; + } + match ScopeAction::from_str(&scope.action) { + Some(ScopeAction::Read) => match job_read_scope_paths(&scope) { + Some(paths) => push_read_paths(paths, &mut exact, &mut prefix), + None => return crate::ScopePathFilter::AllowAll, + }, + Some(ScopeAction::Write) => return crate::ScopePathFilter::AllowAll, + _ => continue, + } + } + if scoped { + crate::ScopePathFilter::Restricted { exact, prefix } + } else { + crate::ScopePathFilter::AllowAll + } +} + +/// What a token's job reads are confined to, or `None` when they are not confined to +/// particular runnables. /// /// A run scope is what the trigger UI mints per script or flow and hands to a webhook /// caller / CI job: it may start the runnables it names and follow those runs, so its -/// by-id job reads must stay within what it can start (enforced by -/// `require_job_read_access`). Both the path (`jobs:run:flows:f/team/etl`) and the -/// kind-only (`jobs:run:scripts`, which legacy `jobs:runscript` tokens carry) forms -/// confine, since `ScopeDefinition::includes` already matches a candidate -/// `jobs:run::` against either. +/// by-id job reads must stay within what it can start. Both the path +/// (`jobs:run:flows:f/team/etl`) and the kind-only (`jobs:run:scripts`, which legacy +/// `jobs:runscript` tokens carry) forms confine, since `ScopeDefinition::includes` +/// already matches a candidate `jobs:run::` against either. +/// +/// A `jobs:read:` scope reads the runs of the runnables at those paths and +/// nothing else. The two kinds of grant add up: a job either admits is readable. /// /// Returns `None` — unconfined — when the token is effectively unscoped, or carries a -/// jobs scope that grants job reads in its own right: `jobs:read`/`jobs:write`, or a -/// bare `jobs:run` (it can start anything, so confining its reads to "what it may run" -/// would restrict nothing). -pub fn job_read_run_confinement(scopes: Option<&[String]>) -> Option> { - let mut confinement = Vec::new(); +/// jobs scope that grants every job read in its own right: a `jobs:read` naming no path +/// (or `*`), a bare `jobs:run` (it can start anything, so confining its reads to "what +/// it may run" would restrict nothing), or `jobs:write`, whose paths are ignored on +/// reads as they are on every job write. +pub fn job_read_confinement(scopes: Option<&[String]>) -> Option { + let mut run = Vec::new(); + let mut exact = Vec::new(); + let mut prefix = Vec::new(); + let mut confined = false; for scope in scopes? .iter() .filter(|s| !s.starts_with("if_jobs:filter_tags:")) @@ -972,26 +708,76 @@ pub fn job_read_run_confinement(scopes: Option<&[String]>) -> Option { - confinement.push(scope) + confined = true; + run.push(scope) } + Some(ScopeAction::Read) => { + let Some(paths) = job_read_scope_paths(&scope) else { + return None; + }; + confined = true; + push_read_paths(paths, &mut exact, &mut prefix); + } + // Grants no reads (see `scope_grants_access`), so it neither confines nor + // frees them. + Some(ScopeAction::Cancel) => continue, Some(_) => return None, None => continue, } } + confined.then_some(JobReadConfinement { + run, + read_paths: crate::ScopePathFilter::Restricted { exact, prefix }, + }) +} + +/// The `jobs:cancel:` scopes a token's cancels are confined to, or `None` when +/// they are not confined: an unscoped token, or one whose cancel grant is not +/// path-scoped (`jobs:cancel`, or `jobs:write`, which stays resource-blind here as on +/// every other job write). +/// +/// A job is within the confinement when its own `runnable_path` or that of any of its +/// `parent_job` ancestors matches (see `cancel_confinement_admits`), so a token scoped +/// to a flow can cancel the flow's steps. +pub fn job_cancel_path_confinement(scopes: Option<&[String]>) -> Option> { + let mut confinement = Vec::new(); + for scope in scopes? + .iter() + .filter(|s| !s.starts_with("if_jobs:filter_tags:")) + { + let Ok(scope) = ScopeDefinition::from_scope_string(scope) else { + continue; + }; + if ScopeDomain::from_str(&scope.domain) != Some(ScopeDomain::Jobs) { + continue; + } + match ScopeAction::from_str(&scope.action) { + Some(ScopeAction::Cancel) if scope.resource.is_some() => confinement.push(scope), + Some(ScopeAction::Cancel | ScopeAction::Write) => return None, + _ => continue, + } + } (!confinement.is_empty()).then_some(confinement) } -/// Whether a job that ran `runnable_path` as `kind` (`scripts` or `flows`) is inside a -/// [`job_read_run_confinement`] set. -pub fn run_confinement_admits( - confinement: &[ScopeDefinition], - kind: &str, - runnable_path: &str, -) -> bool { +/// Whether the token holds a `jobs:cancel` scope, path-scoped or not. +pub fn has_job_cancel_grant(scopes: Option<&[String]>) -> bool { + scopes.is_some_and(|scopes| { + scopes.iter().any(|s| { + ScopeDefinition::from_scope_string(s).is_ok_and(|s| { + ScopeDomain::from_str(&s.domain) == Some(ScopeDomain::Jobs) + && ScopeAction::from_str(&s.action) == Some(ScopeAction::Cancel) + }) + }) + }) +} + +/// Whether a job of `runnable_path` is inside a [`job_cancel_path_confinement`] set. +pub fn cancel_confinement_admits(confinement: &[ScopeDefinition], runnable_path: &str) -> bool { let required = ScopeDefinition::new( ScopeDomain::Jobs.as_str(), - ScopeAction::Run.as_str(), - Some(kind), + ScopeAction::Cancel.as_str(), + None, Some(vec![runnable_path.to_string()]), ); confinement.iter().any(|scope| scope.includes(&required)) @@ -1052,6 +838,17 @@ fn scope_grants_access( })); } + // A path-scoped `jobs:read:` is the one job scope this layer does not treat + // as resource-blind: it reaches only the routes whose handlers confine it to those + // paths, so a job read route is refused to it until it is listed there. + if scope_domain == ScopeDomain::Jobs + && scope_action == ScopeAction::Read + && required_action == ScopeAction::Read + && job_read_scope_paths(scope).is_some() + { + return Ok(route_path.is_some_and(is_read_path_scoped_route)); + } + if !scope_action.includes(&required_action) { return Ok(false); } @@ -1085,7 +882,8 @@ fn scope_grants_access( /// from its own super-admin gate, these among them, so each is already readable by every /// authenticated user; each is here because the CLI reads it from a job: /// `automate_username_creation` before creating a user during a git-sync push, `uid` and -/// `hub_base_url` when `u/admin/hub_sync` pulls resource types from the Hub. The other +/// `hub_base_url` when a job runs `wmill hub pull`, as an edited copy of the retired +/// `u/admin/hub_sync` still does (servers sync resource types themselves). The other /// ungated keys have no such caller, so they stay confined — being ungated earns a key /// nothing on its own. Listing a gated key earns it nothing either: `require_super_admin` /// refuses every job token, so `hub_api_secret`, which that pull reads for a private Hub, @@ -1202,6 +1000,61 @@ pub fn scope_for_route(method: &str, path: &str) -> Option { }) } +/// Routes the runtime of a job calls about that job alone: progress, its root id, its +/// resume and approval urls, a workflow-as-code checkpoint or task. A job token restricted +/// by `job_token_scopes` keeps these whatever its scopes, or the job could not run at all, +/// but only for its own job: the job id in the path must be the token's. +pub fn is_own_job_runtime_route(route_path: &str, http_method: &str, job_id: uuid::Uuid) -> bool { + let Some(rest) = route_path.strip_prefix("/api/w/") else { + return false; + }; + let Some((_workspace, rest)) = rest.split_once('/') else { + return false; + }; + let segments: Vec<&str> = rest.split('/').collect(); + let (method_ok, id_index) = match segments.as_slice() { + ["jobs_u", "get" | "get_root_job_id", ..] => (http_method == "GET", 2), + ["job_metrics", "set_progress", ..] => (http_method == "POST", 2), + ["job_metrics", "get_progress", ..] => (http_method == "GET", 2), + ["jobs" | "jobs_u", "resume_urls" | "wac_approval_urls", ..] => (http_method == "GET", 2), + ["jobs", "worker_is_draining", ..] => (http_method == "GET", 2), + ["jobs", "wac", "inline_checkpoint", ..] => (http_method == "POST", 3), + ["jobs", "run", "workflow_as_code", ..] => (http_method == "POST", 3), + _ => return false, + }; + method_ok + && segments + .get(id_index) + .and_then(|id| uuid::Uuid::parse_str(id).ok()) + .is_some_and(|id| id == job_id) +} + +/// Routes reading the state of a flow run: step results (`results.x` in input transforms, +/// loop and branch results) and the run's user state. The flow orchestrator evaluates a +/// step's inputs with the token of the step that just finished, and the SDK reads user state +/// at the root job, so a restricted job token keeps these for every job of its own run. +/// Returns the job id the path names; the caller checks it against the token's lineage. +pub fn flow_run_read_route_job(route_path: &str, http_method: &str) -> Option { + let rest = route_path.strip_prefix("/api/w/")?; + let (_workspace, rest) = rest.split_once('/')?; + let segments: Vec<&str> = rest.split('/').collect(); + let id = match segments.as_slice() { + ["jobs" | "jobs_u", "result_by_id", id, ..] if http_method == "GET" => id, + ["jobs_u", "completed", "get_result" | "get_result_maybe", id, ..] + if http_method == "GET" => + { + id + } + ["jobs" | "jobs_u", "flow", "user_states", id, ..] + if http_method == "GET" || http_method == "POST" => + { + id + } + _ => return None, + }; + uuid::Uuid::parse_str(id).ok() +} + /// Helper function to check if scopes allow access to a route pub fn check_scopes_for_route( token_scopes: Option<&[String]>, @@ -1221,6 +1074,63 @@ pub fn check_scopes_for_route( mod tests { use super::*; + #[test] + fn own_job_runtime_routes_admit_only_the_tokens_job() { + let own = uuid::Uuid::new_v4(); + let other = uuid::Uuid::new_v4(); + let ok = |path: &str, method: &str| is_own_job_runtime_route(path, method, own); + assert!(ok( + &format!("/api/w/ws/job_metrics/set_progress/{own}"), + "POST" + )); + assert!(ok( + &format!("/api/w/ws/jobs_u/get_root_job_id/{own}"), + "GET" + )); + assert!(ok(&format!("/api/w/ws/jobs/resume_urls/{own}/0"), "GET")); + assert!(ok( + &format!("/api/w/ws/jobs/wac/inline_checkpoint/{own}"), + "POST" + )); + assert!(ok( + &format!("/api/w/ws/jobs/run/workflow_as_code/{own}/main"), + "POST" + )); + assert!(!ok( + &format!("/api/w/ws/job_metrics/set_progress/{other}"), + "POST" + )); + assert!(!ok(&format!("/api/w/ws/jobs_u/get/{own}"), "POST")); + assert!(!ok(&format!("/api/w/ws/jobs/run/p/{own}"), "POST")); + assert!(!ok("/api/w/ws/variables/get_value/u/admin/secret", "GET")); + let run = |path: &str, method: &str| flow_run_read_route_job(path, method); + assert_eq!( + run(&format!("/api/w/ws/jobs/result_by_id/{other}/b"), "GET"), + Some(other) + ); + assert_eq!( + run( + &format!("/api/w/ws/jobs_u/completed/get_result/{other}"), + "GET" + ), + Some(other) + ); + assert_eq!( + run( + &format!("/api/w/ws/jobs/flow/user_states/{other}/k"), + "POST" + ), + Some(other) + ); + assert_eq!( + run( + &format!("/api/w/ws/jobs_u/completed/delete/{other}"), + "POST" + ), + None + ); + } + #[test] fn test_scope_definition_parsing() { let scope = ScopeDefinition::from_scope_string("jobs:read").unwrap(); @@ -1395,28 +1305,16 @@ mod tests { #[test] fn run_scopes_confine_job_reads_by_kind_and_path() { let confinement = - job_read_run_confinement(Some(&["jobs:run:flows:f/team/*".to_string()])).unwrap(); - assert!(run_confinement_admits(&confinement, "flows", "f/team/etl")); + job_read_confinement(Some(&["jobs:run:flows:f/team/*".to_string()])).unwrap(); + assert!(confinement.admits("flows", "f/team/etl")); // Right path, wrong kind — a script named like the flow is not the flow. - assert!(!run_confinement_admits( - &confinement, - "scripts", - "f/team/etl" - )); - assert!(!run_confinement_admits( - &confinement, - "flows", - "f/other/etl" - )); + assert!(!confinement.admits("scripts", "f/team/etl")); + assert!(!confinement.admits("flows", "f/other/etl")); // A kind-only scope confines to that kind, at any path. - let kind_only = job_read_run_confinement(Some(&["jobs:run:scripts".to_string()])).unwrap(); - assert!(run_confinement_admits( - &kind_only, - "scripts", - "u/admin/anything" - )); - assert!(!run_confinement_admits(&kind_only, "flows", "f/team/etl")); + let kind_only = job_read_confinement(Some(&["jobs:run:scripts".to_string()])).unwrap(); + assert!(kind_only.admits("scripts", "u/admin/anything")); + assert!(!kind_only.admits("flows", "f/team/etl")); // Scopes that grant job reads in their own right leave reads unconfined. for scopes in [ @@ -1429,11 +1327,111 @@ mod tests { vec!["if_jobs:filter_tags:deno".to_string()], ] { assert!( - job_read_run_confinement(Some(&scopes)).is_none(), + job_read_confinement(Some(&scopes)).is_none(), "{scopes:?} must not confine job reads" ); } - assert!(job_read_run_confinement(None).is_none()); + assert!(job_read_confinement(None).is_none()); + } + + #[test] + fn jobs_read_paths_confine_reads_and_reach_only_confined_routes() { + let scopes = |s: &[&str]| s.iter().map(|s| s.to_string()).collect::>(); + let scoped = scopes(&["jobs:read:f/served/*,u/svc/report"]); + let confinement = job_read_confinement(Some(&scoped)).unwrap(); + // Any kind of runnable at a named path. + assert!(confinement.admits("flows", "f/served/etl")); + assert!(confinement.admits("scripts", "u/svc/report")); + assert!(!confinement.admits("scripts", "f/other/etl")); + assert!(!confinement.admits("scripts", "f/served_not/etl")); + let lists = |s: &[&str], path: &str| job_list_path_filter(Some(&scopes(s))).allows(path); + assert!(lists( + &["jobs:read:f/served/*,u/svc/report"], + "f/served/etl" + )); + assert!(!lists( + &["jobs:read:f/served/*,u/svc/report"], + "f/other/etl" + )); + + // A run scope adds the runnables it may start to the by-id reads, and nothing to + // the lists, not even a bare one that leaves the by-id reads unconfined. + let both = scopes(&["jobs:read:f/served/*", "jobs:run:scripts:f/other/etl"]); + let confinement = job_read_confinement(Some(&both)).unwrap(); + assert!(confinement.admits("scripts", "f/other/etl")); + assert!(!confinement.admits("flows", "f/other/etl")); + for run in ["jobs:run:scripts:f/other/etl", "jobs:run"] { + assert!( + !lists(&["jobs:read:f/served/*", run], "f/other/etl"), + "{run}" + ); + } + for unfiltered in [ + &["jobs:read"][..], + &["jobs:read:f/served/*", "jobs:write"], + &["if_jobs:filter_tags:deno"], + ] { + assert!(lists(unfiltered, "f/other/etl"), "{unfiltered:?}"); + } + + for unconfined in [ + &["jobs:read:*"][..], + &["jobs:read:f/served/*", "jobs:read"], + &["jobs:read:f/served/*", "jobs:write"], + &["jobs:write:f/served/*"], + ] { + assert!( + job_read_confinement(Some(&scopes(unconfined))).is_none(), + "{unconfined:?}" + ); + } + + let id = "0190f4c2-0000-7000-8000-000000000000"; + for route in [ + format!("jobs_u/get/{id}"), + format!("jobs_u/completed/get_result/{id}"), + format!("jobs_u/getupdate_sse/{id}"), + format!("jobs_u/flow/approval_info/{id}"), + "jobs/list".to_string(), + "jobs/completed/list".to_string(), + "jobs/queue/list".to_string(), + ] { + let route = format!("/api/w/ws/{route}"); + assert!( + check_route_access(&scoped, &route, "GET").is_ok(), + "{route}" + ); + } + // Reads that aggregate over the workspace, resolve past the job they name, or + // mint a credential stay with a `jobs:read` that names no path. + for route in [ + "jobs/list_filtered_uuids".to_string(), + "jobs/queue/list_filtered_uuids".to_string(), + "jobs/queue/count".to_string(), + "jobs/completed/count".to_string(), + "jobs/completed/count_jobs".to_string(), + "jobs/completed/export".to_string(), + "jobs/queue/export".to_string(), + "jobs/asset_dispatch_edges".to_string(), + format!("jobs/queue/scheduled_for/{id}"), + format!("jobs/result_by_id/{id}/a"), + format!("jobs/job_view_token/{id}"), + format!("jobs_u/get_root_job_id/{id}"), + ] { + let route = format!("/api/w/ws/{route}"); + assert!( + check_route_access(&scoped, &route, "GET").is_err(), + "{route}" + ); + assert!( + check_route_access(&scopes(&["jobs:read"]), &route, "GET").is_ok(), + "{route}" + ); + assert!( + check_route_access(&scopes(&["jobs:read:*"]), &route, "GET").is_ok(), + "{route}" + ); + } } #[test] @@ -1737,6 +1735,24 @@ mod tests { scope_for_route("POST", "/api/w/ws/jobs/run/agent/u/x/y").as_deref(), Some("jobs:run:agents") ); + for path in [ + "/api/w/ws/jobs/run/dependencies", + "/api/w/ws/jobs/run/dependencies_async", + "/api/w/ws/jobs/run/flow_dependencies", + "/api/w/ws/jobs/run/flow_dependencies_async", + ] { + assert_eq!( + scope_for_route("POST", path).as_deref(), + Some("jobs:run:dependencies"), + "{path}" + ); + assert!(check_route_access(&["jobs:run:dependencies".into()], path, "POST").is_ok()); + assert!(check_route_access(&["jobs:run:flows".into()], path, "POST").is_err()); + } + // A dependencies grant reaches no other run route. + for path in ["/api/w/ws/jobs/run/preview", "/api/w/ws/jobs/run/p/u/x/y"] { + assert!(check_route_access(&["jobs:run:dependencies".into()], path, "POST").is_err()); + } // Preview/bundle runs have no deployed path and their handlers require the // broad `jobs:run` scope, so the derived scope must not carry a kind. @@ -1781,4 +1797,68 @@ mod tests { // Unknown route -> None so the caller fails closed. assert!(scope_for_route("GET", "/healthz").is_none()); } + + #[test] + fn jobs_cancel_grants_only_the_cancel_routes() { + let cancel = vec!["jobs:cancel:f/served/*".to_string()]; + let id = "0190f4c2-0000-7000-8000-000000000000"; + for route in [ + format!("/api/w/ws/jobs_u/queue/cancel/{id}"), + format!("/api/w/ws/jobs_u/queue/force_cancel/{id}"), + "/api/w/ws/jobs_u/queue/cancel_persistent/f/served/s".to_string(), + "/api/w/ws/jobs/queue/cancel_selection".to_string(), + ] { + assert!( + check_route_access(&cancel, &route, "POST").is_ok(), + "{route}" + ); + assert!( + check_route_access(&["jobs:write".to_string()], &route, "POST").is_ok(), + "{route}" + ); + assert!( + check_route_access(&["jobs:read".to_string()], &route, "POST").is_err(), + "{route}" + ); + } + for (route, method) in [ + (format!("/api/w/ws/jobs_u/get/{id}"), "GET"), + ("/api/w/ws/jobs/list".to_string(), "GET"), + (format!("/api/w/ws/jobs/flow/resume_suspended/{id}"), "POST"), + (format!("/api/w/ws/jobs/queue/run_now/{id}"), "POST"), + ("/api/w/ws/jobs/run/p/f/served/s".to_string(), "POST"), + ] { + assert!( + check_route_access(&cancel, &route, method).is_err(), + "{route}" + ); + } + } + + #[test] + fn jobs_cancel_path_confinement() { + let scopes = |s: &[&str]| s.iter().map(|s| s.to_string()).collect::>(); + let conf = + job_cancel_path_confinement(Some(&scopes(&["jobs:cancel:f/served/*,u/svc/kill_me"]))) + .unwrap(); + assert!(cancel_confinement_admits(&conf, "f/served/etl")); + assert!(cancel_confinement_admits(&conf, "u/svc/kill_me")); + assert!(!cancel_confinement_admits(&conf, "f/other/etl")); + assert!(!cancel_confinement_admits(&conf, "f/served_not/etl")); + + // A cancel grant that is not path-scoped leaves cancels unconfined. + for s in [ + &["jobs:cancel"][..], + &["jobs:cancel:f/a/*", "jobs:write"], + &[], + ] { + assert!( + job_cancel_path_confinement(Some(&scopes(s))).is_none(), + "{s:?}" + ); + } + // A cancel scope neither confines nor frees a run token's reads. + let run = scopes(&["jobs:run:flows:f/a/b", "jobs:cancel"]); + assert!(job_read_confinement(Some(&run)).is_some()); + } } diff --git a/backend/windmill-api-configs/src/lib.rs b/backend/windmill-api-configs/src/lib.rs index 7598021f00..f1f4792e71 100644 --- a/backend/windmill-api-configs/src/lib.rs +++ b/backend/windmill-api-configs/src/lib.rs @@ -224,12 +224,13 @@ async fn update_config( #[cfg(not(feature = "enterprise"))] let config = if name.starts_with("worker__") { - // In CE, only allow setting worker_tags, cache_clear, init_bash, and native_mode + // In CE, only allow setting worker_tags, cache_clear, init_bash, native_mode and paused serde_json::json!({ "worker_tags": config.get("worker_tags"), "cache_clear": config.get("cache_clear"), "init_bash": config.get("init_bash"), - "native_mode": config.get("native_mode") + "native_mode": config.get("native_mode"), + "paused": config.get("paused") }) } else { config @@ -272,6 +273,18 @@ async fn update_config( } let mut tx = db.begin().await?; + let was_paused = if name.starts_with("worker__") { + sqlx::query_scalar!( + "SELECT COALESCE(config->'paused' = 'true'::jsonb, false) FROM config WHERE name = $1 FOR UPDATE", + &name + ) + .fetch_optional(&mut *tx) + .await? + .flatten() + .unwrap_or(false) + } else { + false + }; sqlx::query!( "INSERT INTO config (name, config) VALUES ($1, $2) ON CONFLICT (name) DO UPDATE SET config = EXCLUDED.config", &name, @@ -291,6 +304,17 @@ async fn update_config( ) .await?; tx.commit().await?; + + if name.starts_with("worker__") { + let paused = config.get("paused").and_then(|v| v.as_bool()) == Some(true); + if paused != was_paused { + windmill_common::feature_usage::log_feature_usage( + "worker_group", + "pause_toggle", + if paused { "pause" } else { "resume" }, + ); + } + } Ok(format!("Updated config {name}")) } diff --git a/backend/windmill-api-flows/src/flows.rs b/backend/windmill-api-flows/src/flows.rs index aa86a7ff07..eab73bc015 100644 --- a/backend/windmill-api-flows/src/flows.rs +++ b/backend/windmill-api-flows/src/flows.rs @@ -20,7 +20,8 @@ use windmill_api_auth::{ require_owner_of_path, ApiAuthed, }; use windmill_common::workspaces::{ - check_deploy_rules, check_operator_can_build_flows, operator_can_build_flows, RuleCheckResult, + check_deploy_rules, check_operator_can_build, operator_builder_rights, BuilderKind, + RuleCheckResult, }; use windmill_common::{ user_drafts::{overlay_or_draft_only, DraftUserRef, UserDraftItemKind, WithDraftOverlay}, @@ -243,7 +244,7 @@ async fn list_flows( // Append the authed user's drafts at paths with no deployed flow; see scripts.rs. if lq.include_draft_only.unwrap_or(false) - && (!authed.is_operator || operator_can_build_flows(&db, &w_id).await?) + && (!authed.is_operator || operator_builder_rights(&db, &w_id).await?.flows) && offset == 0 && lq.path_start.is_none() && lq.path_exact.is_none() @@ -258,7 +259,8 @@ async fn list_flows( r#"SELECT DISTINCT ON (path) path, value as "value!: sqlx::types::Json>", - created_at + created_at, + email IS NULL as "legacy!" FROM draft WHERE workspace_id = $1 AND typ = 'flow' @@ -315,9 +317,9 @@ async fn list_flows( inherited_labels: None, is_draft: true, draft_path, - // Synthesized rows are the authed user's own draft. + // Owned by nobody when legacy; see scripts.rs. draft_users: Some(sqlx::types::Json(vec![DraftUserRef { - username: Some(authed.username.clone()), + username: (!row.legacy).then(|| authed.username.clone()), }])), }); } @@ -730,7 +732,14 @@ async fn create_flow( Path(w_id): Path, Json(mut nf): Json, ) -> Result<(StatusCode, String)> { - check_operator_can_build_flows(&db, &w_id, authed.is_operator, "create flows").await?; + check_operator_can_build( + &db, + &w_id, + authed.is_operator, + BuilderKind::Flows, + "create flows", + ) + .await?; check_scopes(&authed, || format!("flows:write:{}", nf.path))?; // A `<= 0` flow timeout is "unset", not a 0-second limit that kills every run instantly. @@ -803,6 +812,24 @@ async fn create_flow( check_schedule_conflict(&mut tx, &w_id, &nf.path).await?; let schema_str = nf.schema.and_then(|x| serde_json::to_string(&x.0).ok()); + let job_token_scopes = nf + .job_token_scopes + .as_ref() + .and_then(|scopes| scopes.as_deref()) + .map(windmill_common::scopes::validate_job_token_scopes) + .transpose()?; + let restricts_steps = windmill_common::scopes::validate_flow_step_job_token_scopes( + &serde_json::from_str::(nf.value.get()) + .map_err(|e| windmill_common::error::Error::BadRequest(e.to_string()))?, + )?; + if job_token_scopes.is_some() || restricts_steps { + windmill_common::min_version::MIN_VERSION_SUPPORTS_JOB_TOKEN_SCOPES + .assert() + .await?; + } + if restricts_steps { + windmill_common::feature_usage::log_feature_usage("job_token_scopes", "deploy", "step:set"); + } let resolved_on_behalf_of = windmill_common::resolve_on_behalf_of( nf.on_behalf_of_email.as_deref(), nf.on_behalf_of.as_deref(), @@ -823,14 +850,14 @@ async fn create_flow( dedicated_worker, visible_to_runner_only, ws_error_handler_muted, value, schema, edited_by, edited_at, labels, - on_behalf_of, on_behalf_of_email + on_behalf_of, on_behalf_of_email, job_token_scopes ) VALUES ( $1, $2, $3, $4, NULL, '', $5, $6, $7, $8, $9, $10::text::json, $11, now(), $12, - $13, $14 + $13, $14, $15 )"#, w_id, nf.path, @@ -846,9 +873,11 @@ async fn create_flow( nf.labels.as_deref() as Option<&[String]>, resolved_on_behalf_of, legacy_on_behalf_of_email, + job_token_scopes.as_deref() as Option<&[String]>, ) .execute(&mut *tx) .await?; + windmill_common::scopes::log_job_token_scopes_deploy("flow", job_token_scopes.as_deref()); let version = sqlx::query_scalar!( "INSERT INTO flow_version (workspace_id, path, value, schema, created_by) @@ -973,6 +1002,7 @@ async fn create_flow( None, None, None, + None, ) .await?; @@ -1152,7 +1182,7 @@ async fn get_flow_version( let mut tx = user_db.begin(&authed).await?; let flow = sqlx::query_as::<_, Flow>( - "SELECT flow.workspace_id, flow.path, flow.summary, flow.description, flow.archived, flow.extra_perms, flow.dedicated_worker, flow.tag, flow.ws_error_handler_muted, flow.timeout, flow.visible_to_runner_only, flow.on_behalf_of, flow.labels, flow_version.schema, flow_version.value, flow_version.created_at as edited_at, flow_version.created_by as edited_by + "SELECT flow.workspace_id, flow.path, flow.summary, flow.description, flow.archived, flow.extra_perms, flow.dedicated_worker, flow.tag, flow.ws_error_handler_muted, flow.timeout, flow.visible_to_runner_only, flow.on_behalf_of, flow.labels, flow.job_token_scopes, flow_version.schema, flow_version.value, flow_version.created_at as edited_at, flow_version.created_by as edited_by FROM flow LEFT JOIN flow_version ON flow_version.path = flow.path AND flow_version.workspace_id = flow.workspace_id WHERE flow.path = $1 AND flow.workspace_id = $2 AND flow_version.id = $3", @@ -1212,6 +1242,7 @@ async fn get_flow_version_by_id( flow.visible_to_runner_only, flow.on_behalf_of, flow.labels, + flow.job_token_scopes, flow_version.schema, flow_version.value, flow_version.created_at as edited_at, @@ -1321,7 +1352,14 @@ async fn update_flow( Path((w_id, flow_path)): Path<(String, StripPath)>, Json(ef): Json, ) -> Result { - check_operator_can_build_flows(&db, &w_id, authed.is_operator, "update flows").await?; + check_operator_can_build( + &db, + &w_id, + authed.is_operator, + BuilderKind::Flows, + "update flows", + ) + .await?; let flow_path = flow_path.to_path(); // The URL identifies the flow being updated; the body path is only needed to rename. let mut nf = ef.into_new_flow(flow_path); @@ -1367,6 +1405,27 @@ async fn update_flow( let old_dep_job = not_found_if_none(old_dep_job, "Flow", flow_path)?; let is_new_path = nf.path != flow_path; let schema_str = schema.and_then(|x| serde_json::to_string(&x).ok()); + // Absent keeps the deployed value: a client unaware of the setting must not drop a + // restriction by saving the flow. + let set_job_token_scopes = nf.job_token_scopes.is_some(); + let job_token_scopes = nf + .job_token_scopes + .as_ref() + .and_then(|scopes| scopes.as_deref()) + .map(windmill_common::scopes::validate_job_token_scopes) + .transpose()?; + let restricts_steps = windmill_common::scopes::validate_flow_step_job_token_scopes( + &serde_json::from_str::(nf.value.get()) + .map_err(|e| windmill_common::error::Error::BadRequest(e.to_string()))?, + )?; + if job_token_scopes.is_some() || restricts_steps { + windmill_common::min_version::MIN_VERSION_SUPPORTS_JOB_TOKEN_SCOPES + .assert() + .await?; + } + if restricts_steps { + windmill_common::feature_usage::log_feature_usage("job_token_scopes", "deploy", "step:set"); + } let resolved_on_behalf_of = windmill_common::resolve_on_behalf_of( nf.on_behalf_of_email.as_deref(), nf.on_behalf_of.as_deref(), @@ -1401,7 +1460,8 @@ async fn update_flow( edited_at = now(), labels = COALESCE($13, labels), on_behalf_of = $14, - on_behalf_of_email = $15 + on_behalf_of_email = $15, + job_token_scopes = CASE WHEN $16 THEN $17 ELSE job_token_scopes END WHERE path = $11 AND workspace_id = $12", if is_new_path { flow_path } else { &nf.path }, @@ -1419,19 +1479,24 @@ async fn update_flow( nf.labels.as_deref() as Option<&[String]>, resolved_on_behalf_of, legacy_on_behalf_of_email, + set_job_token_scopes, + job_token_scopes.as_deref() as Option<&[String]>, ) .execute(&mut *tx) .await .map_err(|e| { error::Error::internal_err(format!("Error updating flow due to flow update: {e:#}")) })?; + if set_job_token_scopes { + windmill_common::scopes::log_job_token_scopes_deploy("flow", job_token_scopes.as_deref()); + } if is_new_path { // if new path, must clone flow to new path and delete old flow for flow_version foreign key constraint sqlx::query!( "INSERT INTO flow - (workspace_id, path, summary, description, archived, extra_perms, dependency_job, tag, ws_error_handler_muted, dedicated_worker, timeout, visible_to_runner_only, on_behalf_of, on_behalf_of_email, concurrency_key, versions, value, schema, edited_by, edited_at, labels) - SELECT workspace_id, $1, summary, description, archived, extra_perms, dependency_job, tag, ws_error_handler_muted, dedicated_worker, timeout, visible_to_runner_only, on_behalf_of, on_behalf_of_email, concurrency_key, versions, value, schema, edited_by, edited_at, labels + (workspace_id, path, summary, description, archived, extra_perms, dependency_job, tag, ws_error_handler_muted, dedicated_worker, timeout, visible_to_runner_only, on_behalf_of, on_behalf_of_email, concurrency_key, versions, value, schema, edited_by, edited_at, labels, job_token_scopes) + SELECT workspace_id, $1, summary, description, archived, extra_perms, dependency_job, tag, ws_error_handler_muted, dedicated_worker, timeout, visible_to_runner_only, on_behalf_of, on_behalf_of_email, concurrency_key, versions, value, schema, edited_by, edited_at, labels, job_token_scopes FROM flow WHERE path = $2 AND workspace_id = $3", nf.path, @@ -1695,6 +1760,7 @@ async fn update_flow( None, None, None, + None, ) .await?; @@ -1870,6 +1936,7 @@ async fn get_flow_by_path( flow.visible_to_runner_only, flow.on_behalf_of, flow.labels, + flow.job_token_scopes, folder_labels(flow.workspace_id, flow.path) AS inherited_labels, flow_version.id AS version_id, flow_version.schema, @@ -1911,6 +1978,7 @@ async fn get_flow_by_path( flow.visible_to_runner_only, flow.on_behalf_of, flow.labels, + flow.job_token_scopes, folder_labels(flow.workspace_id, flow.path) AS inherited_labels, flow_version.id AS version_id, flow_version.schema, @@ -1985,7 +2053,14 @@ async fn archive_flow_by_path( Path((w_id, path)): Path<(String, StripPath)>, Json(archived): Json, ) -> Result { - check_operator_can_build_flows(&db, &w_id, authed.is_operator, "archive flows").await?; + check_operator_can_build( + &db, + &w_id, + authed.is_operator, + BuilderKind::Flows, + "archive flows", + ) + .await?; let path = path.to_path(); check_scopes(&authed, || format!("flows:write:{}", path))?; if let RuleCheckResult::Blocked(msg) = check_deploy_rules( @@ -2126,7 +2201,14 @@ async fn delete_flow_by_path( Path((w_id, path)): Path<(String, StripPath)>, Query(query): Query, ) -> Result { - check_operator_can_build_flows(&db, &w_id, authed.is_operator, "delete flows").await?; + check_operator_can_build( + &db, + &w_id, + authed.is_operator, + BuilderKind::Flows, + "delete flows", + ) + .await?; let path = path.to_path(); check_scopes(&authed, || format!("flows:write:{}", path))?; if let RuleCheckResult::Blocked(msg) = check_deploy_rules( @@ -2335,6 +2417,7 @@ mod tests { apply_preprocessor: None, pass_flow_input_directly: None, debouncing: None, + job_token_scopes: None, }, FlowModule { id: "b".to_string(), @@ -2370,6 +2453,7 @@ mod tests { apply_preprocessor: None, pass_flow_input_directly: None, debouncing: None, + job_token_scopes: None, }, FlowModule { id: "c".to_string(), @@ -2405,6 +2489,7 @@ mod tests { apply_preprocessor: None, pass_flow_input_directly: None, debouncing: None, + job_token_scopes: None, }, ], failure_module: Some(Box::new(FlowModule { @@ -2439,6 +2524,7 @@ mod tests { apply_preprocessor: None, pass_flow_input_directly: None, debouncing: None, + job_token_scopes: None, })), preprocessor_module: None, same_worker: false, diff --git a/backend/windmill-api-groups/src/groups.rs b/backend/windmill-api-groups/src/groups.rs index 0209af8585..d9550d2f5b 100644 --- a/backend/windmill-api-groups/src/groups.rs +++ b/backend/windmill-api-groups/src/groups.rs @@ -470,26 +470,16 @@ async fn update_igroup( require_super_admin(&db, &authed).await?; let mut tx: Transaction<'_, Postgres> = db.begin().await?; - let exists_opt = sqlx::query("SELECT 1 FROM instance_group WHERE name = $1") - .bind(name.clone()) - .fetch_optional(&mut *tx) - .await?; - not_found_if_none(exists_opt, "instance_group", name.clone())?; + let old_role_opt = sqlx::query_scalar!( + "SELECT instance_role FROM instance_group WHERE name = $1 FOR UPDATE", + &name + ) + .fetch_optional(&mut *tx) + .await?; + let old_role = not_found_if_none(old_role_opt, "instance_group", name.clone())?; let validated_role = validate_instance_role(&igroup_update.instance_role)?; - // Fetch old role before updating so we can detect changes - let old_role = if igroup_update.instance_role.is_some() { - sqlx::query_scalar!( - "SELECT instance_role FROM instance_group WHERE name = $1", - &name - ) - .fetch_one(&mut *tx) - .await? - } else { - None - }; - sqlx::query("UPDATE instance_group SET summary = $1, instance_role = $2 WHERE name = $3") .bind(igroup_update.new_summary) .bind(&validated_role) @@ -497,8 +487,9 @@ async fn update_igroup( .execute(&mut *tx) .await?; - // If instance_role actually changed, propagate to all group members - if igroup_update.instance_role.is_some() && old_role != validated_role { + // An omitted or null instance_role is stored as NULL above, so it revokes like any other + // change: members must be recomputed whenever the stored role differs. + if old_role != validated_role { propagate_instance_group_roles(&name, &mut tx).await?; } @@ -664,21 +655,19 @@ async fn delete_igroup( // FOR UPDATE: the group row is the group-level mutex, taken before the workspace // advisory locks (see reconcile_workspace_instance_groups). - let group_role = sqlx::query_scalar!( + sqlx::query_scalar!( "SELECT instance_role FROM instance_group WHERE name = $1 FOR UPDATE", &name ) .fetch_optional(&mut *tx) - .await? - .flatten(); + .await?; - let affected_members: Vec = if group_role.is_some() { + // Collected whatever the group's current role: a member can still hold a role this group + // granted before it was cleared. + let affected_members: Vec = sqlx::query_scalar!("SELECT email FROM email_to_igroup WHERE igroup = $1", &name) .fetch_all(&mut *tx) - .await? - } else { - vec![] - }; + .await?; // Captured and advisory-locked before the settings update strips the group from them. #[cfg(feature = "private")] diff --git a/backend/windmill-api-integration-tests/tests/datatable_roles.rs b/backend/windmill-api-integration-tests/tests/datatable_roles.rs index ddb94f8c2b..023d9f20ed 100644 --- a/backend/windmill-api-integration-tests/tests/datatable_roles.rs +++ b/backend/windmill-api-integration-tests/tests/datatable_roles.rs @@ -417,6 +417,89 @@ async fn concurrent_role_creations_both_survive(db: Pool) -> anyhow::R outcome } +#[cfg(all(feature = "private", feature = "enterprise"))] +#[sqlx::test(migrations = "../migrations", fixtures("base", "datatable_roles"))] +async fn an_existing_role_is_taken_over_only_when_asked_and_never_with_members( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + let suffix: String = uuid::Uuid::new_v4().simple().to_string()[..8].to_string(); + let plain = format!("wmtest_plain_{suffix}"); + let group = format!("wmtest_group_{suffix}"); + let member = format!("wmtest_member_{suffix}"); + for sql in [ + // An expired password and no connection slot: kept as is, the login Windmill sets is dead. + format!( + "CREATE ROLE \"{plain}\" LOGIN PASSWORD 'theirs' VALID UNTIL '2000-01-01' \ + CONNECTION LIMIT 0" + ), + format!("CREATE ROLE \"{group}\" NOLOGIN"), + format!("CREATE ROLE \"{member}\" LOGIN"), + format!("GRANT \"{group}\" TO \"{member}\""), + ] { + sqlx::query(&sql).execute(&db).await?; + } + + let create = |name: String, take_over: bool| async move { + let resp = authed( + client().post(format!( + "http://localhost:{port}/api/settings/datatable_roles" + )), + "SECRET_TOKEN", + ) + .json(&json!({ "name": name, "take_over": take_over })) + .send() + .await?; + Ok::<_, anyhow::Error>((resp.status().as_u16(), resp.text().await?)) + }; + let outcome = async { + let (status, body) = create(plain.clone(), false).await?; + assert_eq!( + status, 409, + "an existing role is not taken without asking: {body}" + ); + + // The group's member would inherit every grant the data table role gets. + let (status, body) = create(group.clone(), true).await?; + assert_eq!(status, 400, "{body}"); + assert!( + body.contains(&member), + "the refusal names the member: {body}" + ); + + let (status, body) = create(plain.clone(), true).await?; + assert_eq!(status, 200, "{body}"); + let catalog = windmill_common::datatable_roles::read_role_catalog( + &db, + windmill_common::datatable_roles::DatatableRoleCluster::Instance, + ) + .await?; + let taken = catalog + .values() + .find(|r| r.name == plain) + .ok_or_else(|| anyhow::anyhow!("{plain} is not in the catalog: {catalog:?}"))?; + let options = (*db.connect_options()) + .clone() + .username(&plain) + .password(taken.pwd.as_deref().unwrap_or_default()); + let mut login = ::connect_with(&options).await?; + sqlx::query("SELECT 1").execute(&mut login).await?; + Ok::<_, anyhow::Error>(()) + } + .await; + + // Roles are cluster-wide and outlive this test's database, whatever happened above. + for name in [&member, &group, &plain] { + let _ = sqlx::query(&format!("DROP ROLE IF EXISTS \"{name}\"")) + .execute(&db) + .await; + } + outcome +} + #[cfg(all(feature = "private", feature = "enterprise"))] #[sqlx::test(migrations = "../migrations", fixtures("base", "datatable_roles"))] async fn a_role_delete_that_fails_part_way_leaves_the_role_disabled( @@ -1764,3 +1847,222 @@ async fn cleanup_waits_out_a_save_racing_it_for_an_instance_database( assert_eq!(users, vec!["test-workspace".to_string()]); Ok(()) } + +#[cfg(all(feature = "private", feature = "enterprise"))] +#[sqlx::test(migrations = "../migrations", fixtures("base", "datatable_roles"))] +async fn a_connection_naming_no_role_holds_the_default_one_and_the_viewers( + db: Pool, +) -> anyhow::Result<()> { + use windmill_common::workspaces::{get_datatable_resource_from_db, DatatableAccess}; + + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let suffix: String = uuid::Uuid::new_v4().simple().to_string()[..8].to_string(); + let names = [ + format!("wmtest_floor_{suffix}"), + format!("wmtest_emea_{suffix}"), + format!("wmtest_amer_{suffix}"), + ]; + let roles_url = format!("http://localhost:{port}/api/settings/datatable_roles"); + const MEMBER_LOGINS: &str = "SELECT m.rolname::text FROM pg_auth_members am + JOIN pg_roles r ON r.oid = am.roleid JOIN pg_roles m ON m.oid = am.member + WHERE r.rolname = $1 AND m.rolname LIKE 'wm\\_set\\_%'"; + + // The job is the app publisher's in every case: only the viewer the server recorded differs. + let connect = |role: Option<&'static str>, viewer: Option<&'static str>| { + get_datatable_resource_from_db( + &db, + "test-workspace", + "main", + role, + DatatableAccess::PermissionedAs { + permissioned_as: "u/test-user", + email: "test@windmill.dev", + end_user_email: viewer, + }, + ) + }; + let set_permissions = |default_role: String, roles: Value| { + sqlx::query( + "UPDATE workspace_settings + SET datatable = jsonb_set(datatable, '{datatables,main,permissions}', $1) + WHERE workspace_id = 'test-workspace'", + ) + .bind(json!({ "default_role": default_role, "roles": roles })) + .execute(&db) + }; + + let outcome = async { + let mut ids = vec![]; + for name in &names { + let created: Value = authed(client().post(&roles_url), "SECRET_TOKEN") + .json(&json!({ "name": name })) + .send() + .await? + .error_for_status()? + .json() + .await?; + ids.push(created["id"].as_str().unwrap().to_string()); + } + // `test-user-2` is named by both regional roles, one of them through a group, and by + // `admin`; `test-user-3` by one; the workspace admin `test-user` by neither. + let roles = json!({ + "admin": {"tenants": ["u/test-user-2"]}, + (ids[0].as_str()): {"tenants": ["*"]}, + (ids[1].as_str()): {"tenants": ["u/test-user-2", "u/test-user-3"]}, + (ids[2].as_str()): {"tenants": ["g/analysts"]}, + }); + set_permissions(ids[0].clone(), roles.clone()).await?; + sqlx::query( + "INSERT INTO usr_to_group (workspace_id, usr, group_) + VALUES ('test-workspace', 'test-user-2', 'analysts')", + ) + .execute(&db) + .await?; + let member_of = |login: String| async { + let mut roles: Vec = sqlx::query_scalar( + "SELECT r.rolname::text FROM pg_auth_members am + JOIN pg_roles r ON r.oid = am.roleid JOIN pg_roles m ON m.oid = am.member + WHERE m.rolname = $1", + ) + .bind(login) + .fetch_all(&db) + .await?; + roles.sort(); + Ok::<_, anyhow::Error>(roles) + }; + + let all = connect(None, Some("test2@windmill.dev")).await?; + let login = all["user"].as_str().unwrap().to_string(); + assert!(login.starts_with("wm_set_"), "{login}"); + let mut expected = names.to_vec(); + expected.sort(); + assert_eq!( + member_of(login.clone()).await?, + expected, + "the login holds the default role and the viewer's, and not `admin`" + ); + assert_eq!(connect(None, Some("test2@windmill.dev")).await?, all); + + let two = connect(None, Some("test3@windmill.dev")).await?; + let mut expected = names[..2].to_vec(); + expected.sort(); + assert_eq!( + member_of(two["user"].as_str().unwrap().to_string()).await?, + expected + ); + + // An admin may use every role, the one owning the tables included. As a viewer they add + // what names them, which here is nothing; nor does a job with no viewer recorded. + for viewer in [Some("test@windmill.dev"), None] { + assert_eq!(connect(None, viewer).await?["user"], names[0].as_str()); + } + // A named role is that role alone, whoever is viewing. + let named: &'static str = Box::leak(names[1].clone().into_boxed_str()); + assert_eq!( + connect(Some(named), Some("test2@windmill.dev")).await?["user"], + names[1].as_str() + ); + // A default of `admin` is the connection it always was: the viewer's roles are not added. + set_permissions("admin".to_string(), roles.clone()).await?; + let admin = connect(None, Some("test2@windmill.dev")).await?; + assert_eq!(admin, connect(Some("admin"), None).await?); + assert!(!admin["user"].as_str().unwrap().starts_with("wm_")); + + // With no default role the viewer's roles are all there is, the one open to everyone + // included, and a connection left with none is refused rather than resolved as anything. + set_permissions("$none".to_string(), roles.clone()).await?; + assert_eq!( + connect(None, Some("test3@windmill.dev")).await?, + two, + "no default changes nothing for a viewer the roles name" + ); + assert!(connect(None, None).await.is_err()); + // The same holds for a default the job's own identity is not a tenant of. + let mut closed = roles.clone(); + closed[ids[0].as_str()] = json!({"tenants": []}); + set_permissions(ids[0].clone(), closed).await?; + let as_member = |viewer: Option<&'static str>| { + get_datatable_resource_from_db( + &db, + "test-workspace", + "main", + None, + DatatableAccess::PermissionedAs { + permissioned_as: "u/test-user-3", + email: "test3@windmill.dev", + end_user_email: viewer, + }, + ) + }; + assert_eq!( + as_member(Some("test3@windmill.dev")).await?["user"], + names[1].as_str() + ); + assert!(as_member(None).await.is_err()); + set_permissions(ids[0].clone(), roles).await?; + + // A set login nobody connects with any more is dropped, a recent one is kept, and a + // dropped one is made again for the next viewer who needs it. + let exists = || async { + sqlx::query_scalar::<_, bool>( + "SELECT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = $1)", + ) + .bind(&login) + .fetch_one(&db) + .await + }; + windmill_common::datatable_roles::sweep_unused_role_set_logins(&db).await?; + assert!(exists().await?, "a set login in use was swept"); + sqlx::query("UPDATE datatable_role_set SET last_used_at = now() - interval '31 days'") + .execute(&db) + .await?; + windmill_common::datatable_roles::sweep_unused_role_set_logins(&db).await?; + assert!(!exists().await?, "an unused set login was kept"); + assert_eq!( + connect(None, Some("test2@windmill.dev")).await?["user"], + login.as_str() + ); + + // A disabled role must not stay reachable through a login that is a member of it. + authed( + client().post(format!("{roles_url}/{}", ids[2])), + "SECRET_TOKEN", + ) + .json(&json!({ "enabled": false })) + .send() + .await? + .error_for_status()?; + let can_login: bool = + sqlx::query_scalar("SELECT rolcanlogin FROM pg_roles WHERE rolname = $1") + .bind(&login) + .fetch_one(&db) + .await?; + assert!(!can_login, "{login} still logs in with a disabled role"); + assert_eq!( + connect(None, Some("test2@windmill.dev")).await?["user"], + two["user"] + ); + Ok::<_, anyhow::Error>(()) + } + .await; + + // Roles are cluster-wide, so they outlive this test's throwaway database. + for name in &names { + let logins: Vec = sqlx::query_scalar(MEMBER_LOGINS) + .bind(name) + .fetch_all(&db) + .await + .unwrap_or_default(); + for login in logins { + let _ = sqlx::query(&format!("DROP ROLE IF EXISTS \"{login}\"")) + .execute(&db) + .await; + } + let _ = sqlx::query(&format!("DROP ROLE IF EXISTS \"{name}\"")) + .execute(&db) + .await; + } + outcome +} diff --git a/backend/windmill-api-integration-tests/tests/groups.rs b/backend/windmill-api-integration-tests/tests/groups.rs index 26e5caeea1..9928d1c060 100644 --- a/backend/windmill-api-integration-tests/tests/groups.rs +++ b/backend/windmill-api-integration-tests/tests/groups.rs @@ -497,6 +497,72 @@ async fn test_remove_user_from_instance_group_rederives_role( Ok(()) } +/// Clearing an instance group's role, or deleting a group whose role was cleared out from under +/// its members, must take the instance-level role back from them. +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn test_instance_group_role_revoked_on_clear_and_delete( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let global_base = format!("http://localhost:{port}/api/groups"); + let email = "test2@windmill.dev"; + + let is_super_admin = || async { + sqlx::query_scalar::<_, bool>("SELECT super_admin FROM password WHERE email = $1") + .bind(email) + .fetch_one(&db) + .await + }; + + let resp = authed(client().post(format!("{global_base}/create"))) + .json(&json!({ "name": "admins_grp" })) + .send() + .await?; + assert_eq!(resp.status(), 200, "create: {}", resp.text().await?); + let resp = authed(client().post(format!("{global_base}/update/admins_grp"))) + .json(&json!({ "new_summary": "", "instance_role": "superadmin" })) + .send() + .await?; + assert_eq!(resp.status(), 200, "set role: {}", resp.text().await?); + let resp = authed(client().post(format!("{global_base}/adduser/admins_grp"))) + .json(&json!({ "email": email })) + .send() + .await?; + assert_eq!(resp.status(), 200, "adduser: {}", resp.text().await?); + assert!(is_super_admin().await?, "group should grant superadmin"); + + let resp = authed(client().post(format!("{global_base}/update/admins_grp"))) + .json(&json!({ "new_summary": "", "instance_role": null })) + .send() + .await?; + assert_eq!(resp.status(), 200, "update: {}", resp.text().await?); + assert!( + !is_super_admin().await?, + "clearing the group's role must revoke superadmin" + ); + + // A member left elevated while the group's stored role is already NULL. + sqlx::query( + "UPDATE password SET super_admin = true, role_source = 'instance_group' WHERE email = $1", + ) + .bind(email) + .execute(&db) + .await?; + + let resp = authed(client().delete(format!("{global_base}/delete/admins_grp"))) + .send() + .await?; + assert_eq!(resp.status(), 200, "delete: {}", resp.text().await?); + assert!( + !is_super_admin().await?, + "deleting a role-cleared group must revoke superadmin" + ); + + Ok(()) +} + /// An overwrite import that moves a member from a dropped group to a retained one must keep /// their workspace data. /// diff --git a/backend/windmill-api-integration-tests/tests/jobs_authed.rs b/backend/windmill-api-integration-tests/tests/jobs_authed.rs index c3da2f3fb9..2c5d2ee78e 100644 --- a/backend/windmill-api-integration-tests/tests/jobs_authed.rs +++ b/backend/windmill-api-integration-tests/tests/jobs_authed.rs @@ -702,3 +702,51 @@ async fn test_list_filtered_job_uuids_resolved_excludes_queue( Ok(()) } + +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn test_worker_is_draining_follows_the_job_worker(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + let job = Uuid::new_v4(); + sqlx::query( + "INSERT INTO v2_job (id, workspace_id, created_by, permissioned_as, kind, tag, args) + VALUES ($1, 'test-workspace', 'test-user', 'u/test-user', 'script', 'deno', '{}'::jsonb)", + ) + .bind(job) + .execute(&db) + .await?; + sqlx::query( + "INSERT INTO v2_job_queue (id, workspace_id, scheduled_for, tag, running, started_at, worker) + VALUES ($1, 'test-workspace', now(), 'deno', true, now(), 'wk-drain-test')", + ) + .bind(job) + .execute(&db) + .await?; + sqlx::query( + "INSERT INTO worker_ping (worker, worker_instance, ip, worker_group, wm_version) + VALUES ('wk-drain-test', 'host', 'NO IP', 'default', 'v0')", + ) + .execute(&db) + .await?; + + let url = format!("http://localhost:{port}/api/w/test-workspace/jobs/worker_is_draining/{job}"); + let draining = || async { + let resp = authed(client().get(&url)).send().await?; + anyhow::Ok(resp.json::().await?) + }; + assert!(!draining().await?); + + windmill_common::worker::set_worker_draining_query("wk-drain-test", &db).await?; + assert!(draining().await?); + + // A parked job keeps its last worker on the queue row without running on it. + sqlx::query("UPDATE v2_job_queue SET started_at = NULL WHERE id = $1") + .bind(job) + .execute(&db) + .await?; + assert!(!draining().await?); + + Ok(()) +} diff --git a/backend/windmill-api-integration-tests/tests/operator_builder_apps.rs b/backend/windmill-api-integration-tests/tests/operator_builder_apps.rs new file mode 100644 index 0000000000..7033d093f2 --- /dev/null +++ b/backend/windmill-api-integration-tests/tests/operator_builder_apps.rs @@ -0,0 +1,570 @@ +use serde_json::json; +use serial_test::serial; +use sqlx::{Pool, Postgres}; +use windmill_common::workspaces::invalidate_operator_rights_cache; +use windmill_test_utils::*; + +/// Every test here must be `#[serial]`: the rights cache is process-global and keyed by workspace id +/// alone, so parallel tests under this same id read each other's rights. +const WS: &str = "test-workspace"; + +fn client(token: &str) -> reqwest::Client { + let mut headers = reqwest::header::HeaderMap::new(); + headers.insert( + reqwest::header::AUTHORIZATION, + reqwest::header::HeaderValue::from_str(&format!("Bearer {token}")).unwrap(), + ); + reqwest::ClientBuilder::new() + .default_headers(headers) + .build() + .unwrap() +} + +async fn set_builder(db: &Pool, flows: bool, apps: bool) -> anyhow::Result<()> { + sqlx::query( + "UPDATE workspace_settings SET operator_settings = $1::text::jsonb WHERE workspace_id = $2", + ) + .bind(format!( + r#"{{"builder_flows": {flows}, "builder_apps": {apps}}}"# + )) + .bind(WS) + .execute(db) + .await?; + // The rights are read through a process-global 60s cache keyed by workspace id. + invalidate_operator_rights_cache(WS); + Ok(()) +} + +async fn add_script(db: &Pool, hash: i64, path: &str, owner: &str) -> anyhow::Result<()> { + sqlx::query( + "INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by, schema, + summary, description, lock, extra_perms) + VALUES ($1, $2, $3, 'x', 'bun', 'script', $4, '{}', '', '', '', '{}')", + ) + .bind(WS) + .bind(hash) + .bind(path) + .bind(owner) + .execute(db) + .await?; + Ok(()) +} + +/// A full-code app is deployed multipart, the way the editor and the CLI use it. +fn raw_app_form(app: serde_json::Value) -> reqwest::multipart::Form { + reqwest::multipart::Form::new() + .part( + "app", + reqwest::multipart::Part::text(app.to_string()) + .mime_str("application/json") + .unwrap(), + ) + .part( + "js", + reqwest::multipart::Part::text("console.log(1)").file_name("app.js"), + ) +} + +fn raw_app(path: &str, mode: &str, runnable_path: &str) -> reqwest::multipart::Form { + raw_app_form(json!({ + "path": path, + "summary": "", + "value": {"files": {}, "runnables": {"r": { + "name": "r", "type": "runnableByPath", "runType": "script", + "path": runnable_path + }}}, + "policy": {"execution_mode": mode, "triggerables_v2": {}} + })) +} + +fn composition_flow(path: &str) -> serde_json::Value { + json!({ + "path": path, "summary": "", "description": "", "schema": {}, + "value": {"modules": [{ + "id": "a", + "value": {"type": "script", "path": "u/operator/some_script", "input_transforms": {}} + }]} + }) +} + +/// The app half of the builder boundary, plus the proof that the two rights are independent: a +/// gate reading "either" would authorize the kind its workspace never granted, which is the whole +/// point of splitting them. +#[sqlx::test(migrations = "../migrations", fixtures("base", "permissions_test"))] +#[serial] +async fn test_operator_builder_apps_boundary(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let api = format!("http://localhost:{port}/api/w/{WS}"); + let c = client("OPERATOR_TOKEN_1"); + + add_script(&db, 4241, "u/operator/some_script", "operator").await?; + // `permissions_test` gives the operator fixture no rights on `u/alice/**`. + add_script(&db, 4243, "u/alice/private", "alice").await?; + // A version shared with the operator, then archived, and a newer one at the same path that is + // not: the newer one is what runs. + add_script(&db, 4244, "u/alice/reused", "alice").await?; + sqlx::query( + "UPDATE script SET archived = true, extra_perms = '{\"u/operator\": false}', + created_at = now() - interval '1 day' WHERE hash = 4244", + ) + .execute(&db) + .await?; + add_script(&db, 4245, "u/alice/reused", "alice").await?; + + set_builder(&db, false, false).await?; + let resp = c + .post(format!("{api}/apps/create_raw")) + .multipart(raw_app( + "u/operator/a0", + "publisher", + "u/operator/some_script", + )) + .send() + .await?; + assert!( + !resp.status().is_success(), + "an operator without the builder right must not create a full-code app" + ); + + set_builder(&db, true, true).await?; + + // `*_raw_source` compiles caller-supplied sources with a bundler job on a worker. It sits + // beside `create_app_raw`, which builders MAY use, so it is the gate most likely to be opened + // by mistake later. + let resp = c + .post(format!("{api}/apps/create_raw_source")) + .json(&json!({ + "path": "u/operator/a1", "summary": "", + "value": {"files": {"index.ts": "console.log(1)"}, "runnables": {}}, + "policy": {"execution_mode": "publisher"} + })) + .send() + .await?; + assert!( + !resp.status().is_success(), + "a builder must not compile app sources on a worker" + ); + + // Low-code apps carry inline scripts, so they stay shut too. + let resp = c + .post(format!("{api}/apps/create")) + .json(&json!({ + "path": "u/operator/a2", "summary": "", + "value": {"grid": []}, "policy": {"execution_mode": "publisher"} + })) + .send() + .await?; + assert!( + !resp.status().is_success(), + "a builder must not create a low-code app" + ); + + let resp = c + .post(format!("{api}/apps/create_raw")) + .multipart(raw_app("u/operator/a3", "publisher", "u/alice/private")) + .send() + .await?; + assert!( + !resp.status().is_success(), + "a builder must not deploy an app referencing a runnable it cannot read" + ); + + let resp = c + .post(format!("{api}/apps/create_raw")) + .multipart(raw_app("u/operator/a4", "viewer", "u/operator/some_script")) + .send() + .await?; + assert!( + !resp.status().is_success(), + "a builder must not deploy a viewer-mode app: the policy stops bounding what it can invoke" + ); + + // Storage strips a NUL, turning this key into `inlineScript`. + let resp = c + .post(format!("{api}/apps/create_raw")) + .multipart(raw_app_form(json!({ + "path": "u/operator/a_nul", "summary": "", + "value": {"files": {}, "runnables": {"r": { + "type": "inline", + "inline\u{0}Script": {"content": "x", "language": "bun"} + }}}, + "policy": {"execution_mode": "publisher", "triggerables_v2": {}} + }))) + .send() + .await?; + assert_eq!( + resp.status(), + 403, + "a NUL in a key must not hide an inline script: {}", + resp.text().await? + ); + + let resp = c + .post(format!("{api}/apps/create_raw")) + .multipart(raw_app( + "u/operator/a5", + "publisher", + "u/operator/some_script", + )) + .send() + .await?; + assert!( + resp.status().is_success(), + "a builder must be able to deploy a full-code app over readable runnables: {}", + resp.text().await? + ); + let sandbox: Option = sqlx::query_scalar( + "SELECT (policy->>'sandbox')::boolean FROM app WHERE workspace_id = $1 AND path = $2", + ) + .bind(WS) + .bind("u/operator/a5") + .fetch_one(&db) + .await?; + assert_eq!( + sandbox, + Some(true), + "a builder-authored app must be stored sandboxed" + ); + + // `execute_component` looks up `:` with an unrestricted component string, so + // a key with a second colon still resolves at run time and must not slip past validation. + let resp = c + .post(format!("{api}/apps/create_raw")) + .multipart(raw_app_form(json!({ + "path": "u/operator/a6", "summary": "", + "value": {"files": {}, "runnables": {}}, + "policy": {"execution_mode": "publisher", "triggerables_v2": { + "x:y:script/u/alice/private": {"static_inputs": {}, "one_of_inputs": {}} + }} + }))) + .send() + .await?; + assert!( + !resp.status().is_success(), + "a multi-colon triggerable key must not hide an unreadable runnable from validation" + ); + + // The two rights are independent. + set_builder(&db, true, false).await?; + let resp = c + .post(format!("{api}/flows/create")) + .json(&composition_flow("u/operator/flows_only")) + .send() + .await?; + assert!( + resp.status().is_success(), + "flows-only must still create a flow: {}", + resp.text().await? + ); + let resp = c + .post(format!("{api}/apps/create_raw")) + .multipart(raw_app( + "u/operator/flows_only_app", + "publisher", + "u/operator/some_script", + )) + .send() + .await?; + assert!( + !resp.status().is_success(), + "flows-only must not deploy a full-code app" + ); + + set_builder(&db, false, true).await?; + let resp = c + .post(format!("{api}/flows/create")) + .json(&composition_flow("u/operator/apps_only")) + .send() + .await?; + assert!( + !resp.status().is_success(), + "apps-only must not create a flow" + ); + let resp = c + .post(format!("{api}/apps/create_raw")) + .multipart(raw_app( + "u/operator/apps_only_app", + "publisher", + "u/operator/some_script", + )) + .send() + .await?; + assert!( + resp.status().is_success(), + "apps-only must still deploy a full-code app: {}", + resp.text().await? + ); + + // The deploy panel saves access and frontend scopes through the settings-only endpoint. + for (body, expected) in [ + ( + json!({"policy": {"execution_mode": "anonymous", "triggerables_v2": {}}}), + 200, + ), + ( + json!({"policy": {"execution_mode": "publisher", "sandbox": false, "triggerables_v2": {}}}), + 403, + ), + ( + json!({"value": {"files": {}, "runnables": {}}, "policy": {"execution_mode": "publisher"}}), + 403, + ), + ] { + let resp = c + .post(format!("{api}/apps/update/u/operator/apps_only_app")) + .json(&body) + .send() + .await?; + let status = resp.status(); + assert_eq!( + status, + expected, + "settings update {body}: {}", + resp.text().await? + ); + } + let sandbox: Option = sqlx::query_scalar( + "SELECT (policy->>'sandbox')::boolean FROM app WHERE workspace_id = $1 AND path = $2", + ) + .bind(WS) + .bind("u/operator/apps_only_app") + .fetch_one(&db) + .await?; + assert_eq!( + sandbox, + Some(true), + "a settings update must not turn a builder app's sandbox off" + ); + + // The editor runs the runnables a draft names as whoever opens it, and its frontend code + // with their session unless the preview is sandboxed. + for (runnables, policy, expected) in [ + (json!({}), json!({"sandbox": true}), 200), + ( + json!({"a": {"type": "inline", "inlineScript": {"content": "x", "language": "bun"}}}), + json!({"sandbox": true}), + 403, + ), + (json!({}), json!({"sandbox": false}), 403), + ( + json!({}), + json!({"sandbox": true, "execution_mode": "viewer"}), + 403, + ), + (json!({}), json!(null), 403), + ( + json!({}), + json!({"sandbox": true, "on_behalf_of": "u/alice", "on_behalf_of_email": "alice@windmill.dev"}), + 403, + ), + ( + json!({}), + json!({"sandbox": true, "on_behalf_of": "u/operator"}), + 200, + ), + ( + json!({}), + json!({"sandbox": true, "triggerables": { + format!("{}script/u/operator/some_script", "a:".repeat(300)): {} + }}), + 400, + ), + ( + json!({"r": {"type": "runnableByPath", "runType": "script", "path": "u/operator/some_script"}}), + json!({"sandbox": true}), + 200, + ), + ( + json!({"r": {"type": "runnableByPath", "runType": "script", "path": "u/alice/private"}}), + json!({"sandbox": true}), + 403, + ), + ( + json!({"r": {"type": "runnableByPath", "runType": "script", "path": "u/alice/reused"}}), + json!({"sandbox": true}), + 403, + ), + ( + json!({"r": {"type": "path", "runType": "hubscript", "path": "hub/1/x/y"}}), + json!({"sandbox": true}), + 403, + ), + ( + json!({"r": {"type": "path", "runType": "script/u/alice", "path": "u/operator/some_script"}}), + json!({"sandbox": true}), + 400, + ), + ( + json!({"r": {"type": "path", "runType": "script", "path": ["u/alice/private"]}}), + json!({"sandbox": true}), + 400, + ), + ( + json!({"r": {"type": ["path"], "runType": "script", "path": "u/alice/private"}}), + json!({"sandbox": true}), + 400, + ), + ( + json!([{"type": "path", "runType": "script", "path": "u/alice/private"}]), + json!({"sandbox": true}), + 400, + ), + ( + json!({"r": {"type": "path", "runType": "script", "path": "u/operator/some_script", + "fields": {"x": {"type": "static", "value": {"type": "path", "path": "/srv/out"}}}}}), + json!({"sandbox": true}), + 200, + ), + ( + json!({}), + json!({"sandbox": true, "triggerables": {"x:rawscript/abc": {}}}), + 403, + ), + ( + json!({}), + json!({"sandbox": true, "triggerables": {"x:script/u/alice/private": {}}}), + 403, + ), + ( + json!({}), + json!({"sandbox": true, "triggerables_v2": { + "x:script/u/alice/private": {"static_inputs": {}, "one_of_inputs": {}}, + "y:rawscript/abc": {"static_inputs": {}, "one_of_inputs": {}} + }}), + 200, + ), + ] { + let resp = c + .post(format!("{api}/drafts/update/raw_app/u/operator/d1")) + .json(&json!({ + "value": {"files": {}, "runnables": runnables, "summary": "", "policy": policy}, + "force": true + })) + .send() + .await?; + let status = resp.status(); + assert_eq!( + status, + expected, + "raw app draft {runnables} {policy}: {}", + resp.text().await? + ); + } + + // The component preview runs a deployed runnable for a builder; inline code and hub + // scripts are unreviewed code. + for (preview, expected) in [ + (json!({"path": "script/u/operator/some_script"}), 200), + (json!({"path": "script/hub/1/x/y"}), 403), + ( + json!({"raw_code": {"content": "x", "language": "bun"}}), + 403, + ), + ] { + let mut body = json!({"component": "c", "args": {}, "force_viewer_static_fields": {}}); + body.as_object_mut() + .unwrap() + .extend(preview.as_object().unwrap().clone()); + let resp = c + .post(format!( + "{api}/apps_u/execute_component/u/operator/apps_only_app" + )) + .json(&body) + .send() + .await?; + let status = resp.status(); + assert_eq!( + status, + expected, + "component preview {preview}: {}", + resp.text().await? + ); + } + + // Delete and redeploy each serve both kinds; builder rights cover only full-code apps. + let resp = client("SECRET_TOKEN") + .post(format!("{api}/apps/create")) + .json(&json!({ + "path": "u/operator/low_code", "summary": "", + "value": {"grid": []}, "policy": {"execution_mode": "publisher"} + })) + .send() + .await?; + assert!(resp.status().is_success(), "{}", resp.text().await?); + let resp = c + .post(format!("{api}/apps/update_raw/u/operator/low_code")) + .multipart(raw_app_form(json!({ + "policy": {"execution_mode": "publisher", "triggerables_v2": {}} + }))) + .send() + .await?; + assert_eq!( + resp.status(), + 403, + "policy-only update of a low-code app: {}", + resp.text().await? + ); + // Deployment history restores a full-code version with allow_kind_change set. + let resp = c + .post(format!("{api}/apps/update_raw/u/operator/apps_only_app")) + .multipart(raw_app_form(json!({ + "value": {"files": {}, "runnables": {}}, + "policy": {"execution_mode": "publisher", "triggerables_v2": {}}, + "allow_kind_change": true + }))) + .send() + .await?; + assert_eq!( + resp.status(), + 200, + "restore of a full-code version: {}", + resp.text().await? + ); + for (path, expected) in [("low_code", 403), ("apps_only_app", 200)] { + let resp = c + .delete(format!("{api}/apps/delete/u/operator/{path}")) + .send() + .await?; + let status = resp.status(); + assert_eq!(status, expected, "delete {path}: {}", resp.text().await?); + } + + Ok(()) +} + +/// The readability check must not hold its RLS transaction while it takes a second connection, +/// or a one-connection pool (`DATABASE_CONNECTIONS=1`) stalls every builder draft naming a script. +#[sqlx::test(migrations = "../migrations", fixtures("base", "permissions_test"))] +#[serial] +async fn test_operator_builder_app_check_fits_one_connection( + pool_opts: sqlx::postgres::PgPoolOptions, + connect_opts: sqlx::postgres::PgConnectOptions, +) -> anyhow::Result<()> { + initialize_tracing().await; + let db = pool_opts + .max_connections(1) + .acquire_timeout(std::time::Duration::from_secs(5)) + .connect_with(connect_opts) + .await?; + let server = ApiServer::start(db.clone()).await?; + let api = format!("http://localhost:{}/api/w/{WS}", server.addr.port()); + add_script(&db, 4241, "u/operator/some_script", "operator").await?; + set_builder(&db, false, true).await?; + + let resp = client("OPERATOR_TOKEN_1") + .post(format!("{api}/drafts/update/raw_app/u/operator/d1")) + .json(&json!({ + "value": {"files": {}, "summary": "", "runnables": {"r": { + "name": "r", "type": "runnableByPath", "runType": "script", + "path": "u/operator/some_script" + }}, "policy": {"sandbox": true, "triggerables_v2": {}}}, + "force": true + })) + .send() + .await?; + let status = resp.status(); + assert_eq!(status, 200, "{}", resp.text().await?); + Ok(()) +} diff --git a/backend/windmill-api-integration-tests/tests/settings.rs b/backend/windmill-api-integration-tests/tests/settings.rs index 348d945a86..7fd60ed8e1 100644 --- a/backend/windmill-api-integration-tests/tests/settings.rs +++ b/backend/windmill-api-integration-tests/tests/settings.rs @@ -200,3 +200,81 @@ async fn test_alert_job_queue_waiting_in_global_settings(db: Pool) -> Ok(()) } + +async fn read_secret_surfaces( + base: &str, + names: &[&str], +) -> anyhow::Result> { + let mut paths = vec![ + "instance_config".to_string(), + "instance_config/yaml".to_string(), + ]; + #[cfg(feature = "enterprise")] + paths.push("list_global".to_string()); + paths.extend(names.iter().map(|n| format!("global/{n}"))); + let mut out = vec![]; + for path in paths { + let resp = authed(client().get(format!("{base}/{path}"))) + .send() + .await?; + let status = resp.status().as_u16(); + out.push((path, status, resp.text().await?)); + } + Ok(out) +} + +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn test_server_secrets_follow_export_flag(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let secrets = [ + ("jwt_secret", json!("planted-jwt-secret")), + ("rsa_keys", json!({"private_key": "planted-rsa-key"})), + ( + "custom_instance_replication_pwd", + json!("planted-replication-pwd"), + ), + ( + "external_instance_pg_state", + json!({"admin_pwd": "planted-pg-state"}), + ), + ]; + for (name, value) in &secrets { + sqlx::query( + "INSERT INTO global_settings (name, value) VALUES ($1, $2) + ON CONFLICT (name) DO UPDATE SET value = EXCLUDED.value", + ) + .bind(name) + .bind(value) + .execute(&db) + .await?; + } + let names: Vec<&str> = secrets.iter().map(|(n, _)| *n).collect(); + let server = ApiServer::start(db.clone()).await?; + let base = format!("http://localhost:{}/api/settings", server.addr.port()); + + // Default: a full export, so `get-config` and `pull` can migrate an instance. + std::env::remove_var("EXPORT_SERVER_SECRETS"); + for (path, status, body) in read_secret_surfaces(&base, &names).await? { + assert_2xx(status, &body, &path); + assert!( + body.contains("planted-"), + "{path} dropped a server secret: {body}" + ); + } + + std::env::set_var("EXPORT_SERVER_SECRETS", "0"); + let withheld = read_secret_surfaces(&base, &names).await; + std::env::remove_var("EXPORT_SERVER_SECRETS"); + for (path, status, body) in withheld? { + if path.starts_with("global/") { + assert_eq!(status, 400, "GET {path} returned {status}"); + } else { + assert_2xx(status, &body, &path); + } + assert!( + !body.contains("planted-"), + "{path} returned a server secret: {body}" + ); + } + Ok(()) +} diff --git a/backend/windmill-api-integration-tests/tests/workspace_comparison.rs b/backend/windmill-api-integration-tests/tests/workspace_comparison.rs index 11c731d75c..1aa2bc8df2 100644 --- a/backend/windmill-api-integration-tests/tests/workspace_comparison.rs +++ b/backend/windmill-api-integration-tests/tests/workspace_comparison.rs @@ -756,6 +756,13 @@ async fn test_compare_workspaces_trigger_and_schedule(db: Pool) -> any ) .execute(&db) .await?; + // The scheduler's missed-occurrence counters only ever move on the enabled side. + sqlx::query( + "UPDATE schedule SET late_run_streak = 1, missed_occurrences = 2, last_missed_at = NOW() + WHERE workspace_id = 'test-workspace' AND path = 'f/sch/runtime_only'", + ) + .execute(&db) + .await?; // ------ Schedule: config change (script_path) in fork. Should diff. sqlx::query!( @@ -2537,7 +2544,10 @@ async fn test_rename_records_the_vacated_path(db: Pool) -> anyhow::Res .await?; let status = resp.status(); let hash = resp.text().await?; - assert!(status.is_success(), "script deploy failed: {status} — {hash}"); + assert!( + status.is_success(), + "script deploy failed: {status} — {hash}" + ); parent_hash = Some(hash.trim().trim_matches('"').to_string()); } let mut vacated = None; diff --git a/backend/windmill-api-jobs/Cargo.toml b/backend/windmill-api-jobs/Cargo.toml index d3e10205ac..dad6469947 100644 --- a/backend/windmill-api-jobs/Cargo.toml +++ b/backend/windmill-api-jobs/Cargo.toml @@ -15,6 +15,7 @@ enterprise = ["windmill-common/enterprise"] [dependencies] windmill-api-auth.workspace = true windmill-api-sse.workspace = true +windmill-audit.workspace = true windmill-common = { workspace = true, default-features = false } windmill-queue.workspace = true axum.workspace = true diff --git a/backend/windmill-api-jobs/src/execution.rs b/backend/windmill-api-jobs/src/execution.rs index 1a750c38ba..c6c2a1fede 100644 --- a/backend/windmill-api-jobs/src/execution.rs +++ b/backend/windmill-api-jobs/src/execution.rs @@ -23,12 +23,17 @@ use std::sync::Arc; use tokio::sync::RwLock; use uuid::Uuid; use windmill_api_auth::{check_scopes, get_scope_tags, ApiAuthed}; +use windmill_audit::{ + audit_oss::{audit_log_many, AuditAuthorable}, + ActionKind, +}; use windmill_common::{ db::{UserDB, UserDbWithAuthed}, error::{self, Error}, flow_conversations::{ add_message_to_conversation_tx, message_attachments, MessageExtras, MessageType, }, + flows::Retry, get_latest_flow_version_info_for_path, jobs::{ check_tag_available_for_workspace_internal, format_result, script_path_to_payload, @@ -122,9 +127,76 @@ pub async fn drop_unclaimable_run_lineage( Ok(()) } +/// Applies the run's `retry` policy by handing `push` a one-step-flow request, which it +/// materializes as a native retryable `Script`. +pub fn with_run_retry( + run_query: &RunJobQuery, + payload: JobPayload, + args: &PushArgs<'_>, + tag: &Option, +) -> error::Result { + let Some(retry) = run_query.retry.as_deref() else { + return Ok(payload); + }; + // Retry attempts are always pushed visible to the owner, which would leak a private run. + if run_query.invisible_to_owner.unwrap_or(false) { + return Err(Error::BadRequest( + "retry cannot be combined with invisible_to_owner".to_string(), + )); + } + let retry: Retry = serde_json::from_str(retry) + .map_err(|e| Error::BadRequest(format!("invalid retry policy: {e}")))?; + let JobPayload::ScriptHash { + hash, + path, + cache_ttl, + cache_ignore_s3_path, + language, + priority, + apply_preprocessor, + debouncing_settings, + concurrency_settings, + .. + } = payload + else { + return Err(Error::BadRequest( + "retry is only supported for workspace scripts".to_string(), + )); + }; + // A failed attempt keeps its raw args and the retry re-push skips the preprocessor, so + // every retry would call `main` with un-preprocessed input. + if apply_preprocessor { + return Err(Error::BadRequest( + "retry is not supported for a script with a preprocessor unless skip_preprocessor is set" + .to_string(), + )); + } + Ok(JobPayload::SingleStepFlow { + path, + hash: Some(hash), + flow_version: None, + language: Some(language), + args: HashMap::from(args), + retry: Some(retry), + error_handler_path: None, + error_handler_args: None, + skip_handler: None, + cache_ttl, + cache_ignore_s3_path, + priority, + tag_override: tag.clone(), + trigger_path: None, + apply_preprocessor, + concurrency_settings, + debouncing_settings, + }) +} + /// The jobs of `referenced` that `authed` cannot claim as its own run lineage: anything but the /// token's own job and that job's `parent_job`, `root_job` and `flow_innermost_root_job`, or for -/// a workspace admin anything outside the workspace. +/// a workspace admin anything outside the workspace. A restricted job token never gets the admin +/// latitude: its flow-run routes trust this lineage (`job_in_same_flow_run`), so a claimed +/// unrelated run would escape its scopes. pub async fn unclaimable_run_lineage( db: &DB, w_id: &str, @@ -157,7 +229,8 @@ pub async fn unclaimable_run_lineage( if referenced.is_empty() { return Ok(referenced); } - let in_workspace = if authed.is_admin { + let restricted_job_token = authed.job_id.is_some() && authed.scopes.is_some(); + let in_workspace = if authed.is_admin && !restricted_job_token { sqlx::query_scalar!( "SELECT id FROM v2_job WHERE id = ANY($1) AND workspace_id = $2", &referenced, @@ -172,6 +245,27 @@ pub async fn unclaimable_run_lineage( Ok(referenced) } +/// `WM_END_USER_EMAIL` of a run started through a `/jobs/run*` route. `WM_EMAIL` names the +/// runnable's `on_behalf_of` identity when it has one, so this is the only place the caller's +/// address reaches the job. A job's own `WM_TOKEN` passes on the end user of that job, and a run +/// a trigger fired has no end user. +pub async fn run_end_user_email( + db: &DB, + w_id: &str, + authed: &ApiAuthed, + trigger: Option<&TriggerMetadata>, +) -> error::Result> { + if trigger.is_some() { + return Ok(None); + } + match authed.job_id { + Some(job_id) => Ok(windmill_common::auth::get_job_perms(db, &job_id, w_id) + .await? + .and_then(|p| p.end_user_email)), + None => Ok(Some(authed.email.clone())), + } +} + #[cfg(feature = "enterprise")] pub async fn check_license_key_valid() -> error::Result<()> { use windmill_common::ee_oss::LICENSE_KEY_VALID; @@ -193,11 +287,15 @@ pub async fn check_license_key_valid() -> error::Result<()> { pub async fn cancel_jobs( jobs: Vec, db: &DB, - username: &str, + author: &(impl AuditAuthorable + Sync), w_id: &str, force_cancel: bool, ) -> error::JsonResult> { + let username = author.username(); let mut uuids = vec![]; + // A job that completed before its turn is reported back like the others but was not + // cancelled, so it gets no audit entry. + let mut audited = vec![]; tracing::info!("Cancelling jobs: {:?}", jobs); let mut tx = db.begin().await?; let trivial_jobs = sqlx::query!("INSERT INTO v2_job_completed AS cj @@ -243,7 +341,7 @@ pub async fn cancel_jobs( } match tokio::time::timeout(tokio::time::Duration::from_secs(5), async move { let tx = db.begin().await?; - let (tx, _) = cancel_job( + let (tx, cancelled) = cancel_job( username, None, job_id.clone(), @@ -255,13 +353,14 @@ pub async fn cancel_jobs( ) .await?; tx.commit().await?; - Ok::<_, anyhow::Error>(()) + Ok::<_, anyhow::Error>(cancelled) }) .await { Ok(result) => match result { - Ok(_) => { + Ok(cancelled) => { uuids.push(job_id); + audited.extend(cancelled); } Err(e) => { tracing::error!("Failed to cancel job {:?}: {:?}", job_id, e); @@ -276,8 +375,27 @@ pub async fn cancel_jobs( } } + audited.extend(trivial_jobs.iter().copied()); uuids.extend(trivial_jobs); + // The cancels are committed by now: failing here would report them as not done. + if let Err(e) = audit_log_many( + db, + author, + if force_cancel { + "jobs.force_cancel" + } else { + "jobs.cancel" + }, + ActionKind::Delete, + w_id, + &audited.iter().map(|id| id.to_string()).collect::>(), + ) + .await + { + tracing::error!("Failed to write audit entries for cancelled jobs in {w_id}: {e:#}"); + } + Ok(Json(uuids)) } @@ -825,6 +943,7 @@ pub async fn handle_chat_conversation_messages( tx, memory_id, Some(job_id), + job_id, &user_message, MessageType::User, None, @@ -865,6 +984,7 @@ pub async fn run_flow<'c>( chat_input_enabled, early_return, labels, + job_token_scopes, .. } = flow_version_info; @@ -914,6 +1034,8 @@ pub async fn run_flow<'c>( ) }; + let end_user_email = run_end_user_email(db, w_id, authed, trigger.as_ref()).await?; + let scope_ceiling = windmill_api_auth::caller_scope_ceiling(db, authed).await?; let (uuid, mut tx) = push( &db, tx, @@ -924,6 +1046,7 @@ pub async fn run_flow<'c>( version, apply_preprocessor, labels, + job_token_scopes, }, push_args, authed.display_username(), @@ -947,9 +1070,10 @@ pub async fn run_flow<'c>( None, push_authed.as_ref(), false, - None, + end_user_email, authed.trigger_or_fallback(trigger), run_query.suspended_mode, + scope_ceiling.as_deref(), ) .await?; @@ -1115,6 +1239,7 @@ pub async fn push_script_job_by_path_into_queue<'c>( let tag = run_query.tag.clone().or(tag); let push_args = PushArgs { args: &args.args, extra: args.extra }; check_tag_available_for_workspace(&db, &w_id, &tag, &push_args, &authed).await?; + let job_payload = with_run_retry(&run_query, job_payload, &push_args, &tag)?; let return_tx = tx_o.is_some(); @@ -1141,6 +1266,8 @@ pub async fn push_script_job_by_path_into_queue<'c>( ) }; + let end_user_email = run_end_user_email(&db, &w_id, &authed, trigger.as_ref()).await?; + let scope_ceiling = windmill_api_auth::caller_scope_ceiling(&db, &authed).await?; let (uuid, tx) = push( &db, tx, @@ -1173,9 +1300,10 @@ pub async fn push_script_job_by_path_into_queue<'c>( }, push_authed.as_ref(), false, - None, + end_user_email, authed.trigger_or_fallback(trigger), run_query.suspended_mode, + scope_ceiling.as_deref(), ) .await?; diff --git a/backend/windmill-api-jobs/src/job_metrics.rs b/backend/windmill-api-jobs/src/job_metrics.rs index abb95693f5..ddbe5a047a 100644 --- a/backend/windmill-api-jobs/src/job_metrics.rs +++ b/backend/windmill-api-jobs/src/job_metrics.rs @@ -173,10 +173,33 @@ struct JobProgressSetRequest { } async fn set_job_progress( + authed: windmill_api_auth::ApiAuthed, Extension(db): Extension, Path((w_id, job_id)): Path<(String, Uuid)>, Json(JobProgressSetRequest { percent, flow_job_id }): Json, ) -> error::JsonResult<()> { + // A restricted job token reaches this route without a scope only for its own job + // (`is_own_job_runtime_route`), so the flow it reports progress to must be its own too. + if let (Some(flow_job_id), Some(token_job), Some(_)) = + (flow_job_id, authed.job_id, &authed.scopes) + { + let own_flow = sqlx::query_scalar!( + "SELECT $2 IN (parent_job, root_job, flow_innermost_root_job) FROM v2_job + WHERE id = $1 AND workspace_id = $3", + token_job, + flow_job_id, + &w_id + ) + .fetch_optional(&db) + .await? + .flatten() + .unwrap_or(false); + if !own_flow { + return Err(error::Error::PermissionDenied(format!( + "flow job {flow_job_id} is not this job's flow" + ))); + } + } // If flow_job_id exists, than we should modify flow_status of corresponding module // Individual jobs and flows are handled differently if let Some(flow_job_id) = flow_job_id { diff --git a/backend/windmill-api-jobs/src/jobs_export.rs b/backend/windmill-api-jobs/src/jobs_export.rs index 6389134031..b2b69367bc 100644 --- a/backend/windmill-api-jobs/src/jobs_export.rs +++ b/backend/windmill-api-jobs/src/jobs_export.rs @@ -112,6 +112,8 @@ pub struct ExportableQueuedJob { pub preprocessed: Option, pub args: Option>>, pub labels: Option>, + #[serde(default)] + pub job_token_scopes: Option>, pub pre_run_error: Option, // v2_job_queue columns (excluding workspace_id and id/created_at/tag/priority) @@ -276,6 +278,7 @@ pub async fn export_queued_jobs( v2_job.preprocessed, v2_job.args as "args: _", v2_job.labels, + job_perms.job_token_scopes as "job_token_scopes?", v2_job.pre_run_error, v2_job_queue.started_at, @@ -298,6 +301,7 @@ pub async fn export_queued_jobs( concurrency_key.key as "concurrency_key?" FROM v2_job_queue INNER JOIN v2_job ON v2_job.id = v2_job_queue.id + LEFT JOIN job_perms ON job_perms.job_id = v2_job_queue.id LEFT JOIN v2_job_runtime ON v2_job_runtime.id = v2_job_queue.id LEFT JOIN v2_job_status ON v2_job_status.id = v2_job_queue.id LEFT JOIN concurrency_key ON concurrency_key.job_id = v2_job_queue.id @@ -445,6 +449,23 @@ pub async fn import_queued_jobs( )); } + // Imported rows bypass `push`, so nothing would cap their tokens: a restricted job token + // could otherwise queue arbitrary jobs that run with its owner's full permissions. + if authed.job_id.is_some() && authed.scopes.is_some() { + return Err(error::Error::PermissionDenied( + "A restricted job token cannot import queued jobs".to_string(), + )); + } + // An imported job gets no `job_perms` row, so its token would be minted without the + // restriction it was queued with. + if let Some(job) = jobs.iter().find(|job| job.job_token_scopes.is_some()) { + return Err(error::Error::BadRequest(format!( + "Queued job {} restricts its job token and cannot be imported; run it again on \ + this instance instead", + job.id + ))); + } + let mut tx = user_db.begin(&authed).await?; for job in jobs { @@ -697,7 +718,8 @@ pub async fn delete_jobs( let mut conversation_ids: Vec = sqlx::query_scalar!( "DELETE FROM flow_conversation_message m USING flow_conversation c - WHERE m.conversation_id = c.id AND c.workspace_id = $1 AND m.job_id = ANY($2) + WHERE m.conversation_id = c.id AND c.workspace_id = $1 + AND COALESCE(m.turn_job_id, m.job_id) = ANY($2) RETURNING m.conversation_id", &w_id, &job_ids diff --git a/backend/windmill-api-jobs/src/query.rs b/backend/windmill-api-jobs/src/query.rs index 857b2e02e2..96d8ca3698 100644 --- a/backend/windmill-api-jobs/src/query.rs +++ b/backend/windmill-api-jobs/src/query.rs @@ -26,6 +26,34 @@ fn not_in_nullable(col: &str, quoted: &[String]) -> String { ) } +/// Narrows a job list to the top-level runs of deployed scripts and flows at an `exact` +/// path, or at or under a `prefix` on the `/` boundary (`ScopePathFilter::allows`). +/// +/// Top-level only because every `runnable_path` index on `v2_job` is partial on +/// `parent_job IS NULL`. Deployed kinds only because a preview's `runnable_path` is +/// whatever its caller sent. +pub fn and_where_root_run_at_paths(sqlb: &mut SqlBuilder, exact: &[String], prefix: &[String]) { + let mut clauses = Vec::new(); + let equal: Vec<_> = exact.iter().chain(prefix).map(|p| quote(p)).collect(); + if !equal.is_empty() { + clauses.push(format!("v2_job.runnable_path IN ({})", equal.join(", "))); + } + for p in prefix { + let pattern = format!("{}/%", escape_ilike_pattern(p)); + clauses.push(format!("v2_job.runnable_path LIKE {}", quote(&pattern))); + } + if clauses.is_empty() { + sqlb.and_where("false"); + return; + } + sqlb.and_where_is_null("v2_job.parent_job") + .and_where( + "v2_job.kind IN ('script', 'script_hub', 'unassigned_script', 'flow', \ + 'unassigned_flow', 'singlestepflow', 'unassigned_singlestepflow')", + ) + .and_where(format!("({})", clauses.join(" OR "))); +} + pub fn filter_list_queue_query( mut sqlb: SqlBuilder, lq: &ListQueueQuery, diff --git a/backend/windmill-api-jobs/src/types.rs b/backend/windmill-api-jobs/src/types.rs index f37040bd11..f50cb61451 100644 --- a/backend/windmill-api-jobs/src/types.rs +++ b/backend/windmill-api-jobs/src/types.rs @@ -52,6 +52,8 @@ pub struct RunJobQuery { pub trigger_external_id: Option, pub service_name: Option, pub suspended_mode: Option, + /// A JSON-encoded `Retry` policy for a script run; see `with_run_retry`. + pub retry: Option, } impl RunJobQuery { diff --git a/backend/windmill-api-schedule/src/lib.rs b/backend/windmill-api-schedule/src/lib.rs index 7a523c89fa..b8ee9ded91 100644 --- a/backend/windmill-api-schedule/src/lib.rs +++ b/backend/windmill-api-schedule/src/lib.rs @@ -1487,11 +1487,13 @@ async fn set_default_error_handler( } }; - if let Some(value_content) = value { - windmill_api_settings::set_global_setting_internal(&db, key, value_content).await?; + if let Some(value_content) = value.clone() { + windmill_api_settings::set_global_setting_internal(&db, key.clone(), value_content).await?; } else { windmill_api_settings::delete_global_setting(&db, key.as_str()).await?; } + windmill_audit::instance_config::audit_setting_write(&db, &authed, &key, value.as_ref(), None) + .await?; if payload.override_existing { // The rewrite and its history rows go in one transaction: on separate diff --git a/backend/windmill-api-scripts/src/scripts.rs b/backend/windmill-api-scripts/src/scripts.rs index bce67755bb..4f4c962759 100644 --- a/backend/windmill-api-scripts/src/scripts.rs +++ b/backend/windmill-api-scripts/src/scripts.rs @@ -382,7 +382,8 @@ async fn list_scripts( r#"SELECT DISTINCT ON (path) path, value as "value!: sqlx::types::Json>", - created_at + created_at, + email IS NULL as "legacy!" FROM draft WHERE workspace_id = $1 AND typ = 'script' @@ -463,9 +464,10 @@ async fn list_scripts( inherited_labels: None, is_draft: true, draft_path, - // Synthesized rows are the authed user's own draft (single-user case). + // A legacy (`email IS NULL`) row belongs to nobody: naming the caller would + // route every discard to their own (absent) row and leave it undeletable. draft_users: Some(sqlx::types::Json(vec![DraftUserRef { - username: Some(authed.username.clone()), + username: (!row.legacy).then(|| authed.username.clone()), }])), }); } @@ -897,6 +899,7 @@ async fn is_noop_deploy_against_parent( ns: &NewScript, parent: &Script, resolved_on_behalf_of: Option<&str>, + resolved_job_token_scopes: Option<&[String]>, db: &DB, ) -> Result { if parent.archived || parent.deleted { @@ -950,6 +953,8 @@ async fn is_noop_deploy_against_parent( // caller-intent flag (auto-resolve parent), not script state auto_parent: _, labels, + // resolved against the deployed value into `resolved_job_token_scopes`, compared below + job_token_scopes: _, // caller-intent flag (preserve user drafts on CLI/git-sync deploys); // transient, never persisted, does not change what the script *is* skip_draft_deletion: _, @@ -1023,6 +1028,9 @@ async fn is_noop_deploy_against_parent( if resolved_on_behalf_of != parent.on_behalf_of.as_deref() { return Ok(false); } + if resolved_job_token_scopes != parent.job_token_scopes.as_deref() { + return Ok(false); + } // Both of a dbt script's derived fields are compared as they WOULD BE STORED, // not as they arrived: the schema comes from the descriptor and the clients // cannot derive it (`windmill-parser-wasm` has no dbt arm), so they send the @@ -1490,6 +1498,37 @@ async fn create_script_internal<'c>( parent_adopted_from_retired_path = ns.parent_hash.is_some(); } + // Absent keeps the previous version's value, read once the parent is settled (a rename + // adopts its source head above), so a client unaware of the setting cannot drop a + // restriction by redeploying or renaming. + let resolved_job_token_scopes: Option> = match (&ns.job_token_scopes, &ns.parent_hash) { + (Some(Some(scopes)), _) => { + windmill_common::min_version::MIN_VERSION_SUPPORTS_JOB_TOKEN_SCOPES + .assert() + .await?; + Some(windmill_common::scopes::validate_job_token_scopes(scopes)?) + } + (Some(None), _) => None, + (None, Some(parent_hash)) => sqlx::query_scalar!( + "SELECT job_token_scopes FROM script WHERE hash = $1 AND workspace_id = $2", + parent_hash.0, + &w_id + ) + .fetch_optional(&db) + .await? + .flatten(), + (None, None) => sqlx::query_scalar!( + "SELECT job_token_scopes FROM script WHERE path = $1 AND workspace_id = $2 \ + AND deleted = false ORDER BY created_at DESC LIMIT 1", + &ns.path, + &w_id + ) + .fetch_optional(&db) + .await? + .flatten(), + }; + ns.job_token_scopes = Some(resolved_job_token_scopes.clone()); + // Before hashing, so the hash and the no-op check see the schema that gets stored. // `{}` counts as absent: an agent filling every tool argument sends it for "none". let schema_absent = ns.schema.as_ref().is_none_or(|s| { @@ -1618,8 +1657,14 @@ async fn create_script_internal<'c>( // CLI pushes must not produce phantom commits on the downstream // git repository. if skip_if_noop - && is_noop_deploy_against_parent(&ns, &ps, resolved_on_behalf_of.as_deref(), &db) - .await? + && is_noop_deploy_against_parent( + &ns, + &ps, + resolved_on_behalf_of.as_deref(), + resolved_job_token_scopes.as_deref(), + &db, + ) + .await? { tracing::info!( workspace_id = %w_id, @@ -2195,8 +2240,8 @@ async fn create_script_internal<'c>( content, created_by, schema, is_template, extra_perms, lock, language, kind, tag, \ envs, concurrent_limit, concurrency_time_window_s, cache_ttl, \ dedicated_worker, ws_error_handler_muted, priority, restart_unless_cancelled, \ - delete_after_use, delete_after_secs, timeout, concurrency_key, visible_to_runner_only, auto_kind, codebase, has_preprocessor, schema_validation, assets, debounce_key, debounce_delay_s, cache_ignore_s3_path, runnable_settings_handle, modules, labels, on_behalf_of, on_behalf_of_email) \ - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9::text::json, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26, $27, $28, $29, $30, $31, $32, $33, $34, $35, $36, $37, $38, $39, $40, $41)", + delete_after_use, delete_after_secs, timeout, concurrency_key, visible_to_runner_only, auto_kind, codebase, has_preprocessor, schema_validation, assets, debounce_key, debounce_delay_s, cache_ignore_s3_path, runnable_settings_handle, modules, labels, on_behalf_of, on_behalf_of_email, job_token_scopes) \ + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9::text::json, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26, $27, $28, $29, $30, $31, $32, $33, $34, $35, $36, $37, $38, $39, $40, $41, $42)", &w_id, &hash.0, ns.path, @@ -2240,9 +2285,14 @@ async fn create_script_internal<'c>( ns.labels.as_deref() as Option<&[String]>, resolved_on_behalf_of, legacy_on_behalf_of_email, + resolved_job_token_scopes.as_deref() as Option<&[String]>, ) .execute(&mut *tx) .await?; + windmill_common::scopes::log_job_token_scopes_deploy( + "script", + resolved_job_token_scopes.as_deref(), + ); // A lock that is not left to a dependency job queues none, so this is the only place its hash // can be recorded. `try_skip_relock` treats a missing hash for an imported script as changed, @@ -2952,6 +3002,7 @@ async fn create_script_internal<'c>( None, None, None, + None, ) .await?; @@ -3075,6 +3126,7 @@ async fn create_script_internal<'c>( None, None, None, + None, ) .await?; tracing::info!("pushed auto-build binary job {job_id} for {script_path}"); @@ -4214,12 +4266,6 @@ async fn delete_script_by_path( check_scopes(&authed, || format!("scripts:write:{}", path))?; - if path == "u/admin/hub_sync" && w_id == "admins" { - return Err(Error::BadRequest( - "Cannot delete the global setup app".to_string(), - )); - } - if let RuleCheckResult::Blocked(msg) = check_deploy_rules( &w_id, AuditAuthorable::username(&authed), @@ -4393,12 +4439,6 @@ async fn delete_scripts_bulk( require_admin(authed.is_admin, &authed.username)?; - if request.paths.contains(&"u/admin/hub_sync".to_string()) && w_id == "admins" { - return Err(Error::BadRequest( - "Cannot delete the global setup app".to_string(), - )); - } - if let RuleCheckResult::Blocked(msg) = check_deploy_rules( &w_id, AuditAuthorable::username(&authed), diff --git a/backend/windmill-api-settings/src/lib.rs b/backend/windmill-api-settings/src/lib.rs index 71cbcc8460..1359e8c142 100644 --- a/backend/windmill-api-settings/src/lib.rs +++ b/backend/windmill-api-settings/src/lib.rs @@ -29,6 +29,12 @@ mod log_cleanup; mod storage_usage; use windmill_api_auth::{require_devops_role, require_super_admin, ApiAuthed}; +use windmill_audit::{ + audit_oss::audit_log, + instance_config::{audit_setting_write, InstanceConfigAuditor}, + ActionKind, +}; +use windmill_common::hub_resource_types::{self, SyncSource}; use windmill_common::utils::HTTP_CLIENT_PERMISSIVE as HTTP_CLIENT; use windmill_common::DB; @@ -57,10 +63,10 @@ use windmill_common::{ error::{self, pg_error_message, JsonResult, Result}, get_database_url, global_settings::{ - ACCENT_COLOR_SETTING, AI_CONFIG_SETTING, APP_WORKSPACED_ROUTE_SETTING, - AUTOMATE_USERNAME_CREATION_SETTING, CRITICAL_ALERT_MUTE_UI_SETTING, CUSTOM_TAGS_SETTING, - DEFAULT_TAGS_WORKSPACES_SETTING, DISABLE_HUB_SETTING, EMAIL_DOMAIN_SETTING, ENV_SETTINGS, - EXTERNAL_INSTANCE_PG_SETTING, + ACCENT_COLOR_SETTING, AI_CONFIG_SETTING, API_BASE_URL_SETTING, + APP_WORKSPACED_ROUTE_SETTING, AUTOMATE_USERNAME_CREATION_SETTING, + CRITICAL_ALERT_MUTE_UI_SETTING, CUSTOM_TAGS_SETTING, DEFAULT_TAGS_WORKSPACES_SETTING, + DISABLE_HUB_SETTING, EMAIL_DOMAIN_SETTING, ENV_SETTINGS, EXTERNAL_INSTANCE_PG_SETTING, GITHUB_APP_WEBHOOK_BASE_URL_SETTING, HTTP_ROUTE_DEFAULT_ALLOWED_ORIGINS_SETTING, HTTP_ROUTE_WORKSPACED_ROUTE_SETTING, HUB_ACCESSIBLE_URL_SETTING, HUB_BASE_URL_SETTING, INSTANCE_BANNER_SETTING, MAX_RETENTION_OVERRIDE_WORKSPACES, @@ -829,7 +835,28 @@ pub async fn set_global_setting( Json(value): Json, ) -> error::Result<()> { require_super_admin(&db, &authed).await?; - set_global_setting_internal(&db, key, value.value.unwrap_or(serde_json::Value::Null)).await + let value = value.value.unwrap_or(serde_json::Value::Null); + set_global_setting_internal(&db, key.clone(), value.clone()).await?; + audit_setting_write(&db, &authed, &key, Some(&value), None).await +} + +async fn audit_instance_action( + db: &DB, + authed: &ApiAuthed, + operation: &str, + action_kind: ActionKind, + parameters: Option>, +) -> error::Result<()> { + audit_log( + db, + authed, + operation, + action_kind, + "global", + Some(&authed.email), + parameters, + ) + .await } pub async fn set_global_setting_internal( @@ -1173,26 +1200,22 @@ async fn run_setting_pre_write_hook( } } } - GITHUB_APP_WEBHOOK_BASE_URL_SETTING => { - // A bad value here yields a webhook GitHub can never deliver to, and the + GITHUB_APP_WEBHOOK_BASE_URL_SETTING | API_BASE_URL_SETTING => { + // A bad webhook base yields a webhook GitHub can never deliver to, and the // failure only shows up much later as "falling back to polling" on a - // repository — so reject it at the boundary instead. + // repository; a bad API base puts an unreachable URL in front of every user. + // So reject both at the boundary instead. match value { // Clearing (delete row) is handled by the caller; allow it through. serde_json::Value::Null => {} serde_json::Value::String(s) if s.trim().is_empty() => {} serde_json::Value::String(s) => { - windmill_common::global_settings::validate_webhook_base_url(s).map_err( - |e| { - error::Error::BadRequest(format!( - "{GITHUB_APP_WEBHOOK_BASE_URL_SETTING}: {e}" - )) - }, - )?; + windmill_common::global_settings::validate_webhook_base_url(s) + .map_err(|e| error::Error::BadRequest(format!("{key}: {e}")))?; } _ => { return Err(error::Error::BadRequest(format!( - "{GITHUB_APP_WEBHOOK_BASE_URL_SETTING} must be a URL string" + "{key} must be a URL string" ))); } } @@ -1235,7 +1258,7 @@ async fn get_instance_config( authed: ApiAuthed, ) -> JsonResult { require_super_admin(&db, &authed).await?; - let config = InstanceConfig::from_db(&db) + let config = InstanceConfig::from_db_for_api(&db) .await .map_err(|e| error::Error::internal_err(e.to_string()))?; Ok(Json(config)) @@ -1246,7 +1269,7 @@ async fn get_instance_config_yaml( authed: ApiAuthed, ) -> error::Result { require_super_admin(&db, &authed).await?; - let config = InstanceConfig::from_db(&db) + let config = InstanceConfig::from_db_for_api(&db) .await .map_err(|e| error::Error::internal_err(e.to_string()))?; let yaml = config @@ -1305,13 +1328,15 @@ async fn set_instance_config( run_setting_pre_write_hook(&db, key, value).await?; } } + let auditor = InstanceConfigAuditor { db: &db, author: &authed, source: None }; + // The external-instance write removes its own key from the diff. windmill_common::external_instance_pg::write_external_instance_pg_from_diff( &db, &mut settings_diff, + &auditor, ) .await?; - - instance_config::apply_settings_diff(&db, &settings_diff) + instance_config::apply_settings_diff(&db, &settings_diff, &auditor) .await .map_err(|e| error::Error::internal_err(e.to_string()))?; @@ -1343,9 +1368,13 @@ async fn set_instance_config( .collect(); let configs_diff = instance_config::diff_worker_configs(¤t_wc, &desired_wc, ApplyMode::Merge); - instance_config::apply_configs_diff(&db, &configs_diff) - .await - .map_err(|e| error::Error::internal_err(e.to_string()))?; + instance_config::apply_configs_diff( + &db, + &configs_diff, + &InstanceConfigAuditor { db: &db, author: &authed, source: None }, + ) + .await + .map_err(|e| error::Error::internal_err(e.to_string()))?; } Ok(()) @@ -1386,6 +1415,11 @@ pub async fn get_global_setting( { require_super_admin(&db, &authed).await?; } + if instance_config::is_withheld_server_secret(&key) { + return Err(error::Error::BadRequest(format!( + "{key} is a server secret and this server does not export it (EXPORT_SERVER_SECRETS=false)" + ))); + } let value = sqlx::query!("SELECT value FROM global_settings WHERE name = $1", key) .fetch_optional(&db) .await? @@ -1428,7 +1462,10 @@ async fn list_global_settings( require_super_admin(&db, &authed).await?; let settings = sqlx::query_as!(GlobalSetting, "SELECT name, value FROM global_settings") .fetch_all(&db) - .await?; + .await? + .into_iter() + .filter(|s| !instance_config::is_withheld_server_secret(&s.name)) + .collect(); Ok(Json(settings)) } @@ -1575,6 +1612,14 @@ pub async fn renew_license_key( } ))); } else { + audit_instance_action( + &db, + &authed, + "settings.renew_license_key", + ActionKind::Update, + None, + ) + .await?; return Ok("Renewed license key".to_string()); } } @@ -1804,6 +1849,14 @@ async fn refresh_custom_instance_user_pwd( require_super_admin(&db, &authed).await?; windmill_common::utils::refresh_custom_instance_user_pwd(&db).await?; windmill_common::utils::refresh_custom_instance_replication_user_pwd(&db).await?; + audit_instance_action( + &db, + &authed, + "settings.refresh_custom_instance_user_pwd", + ActionKind::Update, + None, + ) + .await?; Ok(Json(())) } @@ -1962,7 +2015,8 @@ async fn setup_custom_instance_pg_database( windmill_common::datatable_roles::lock_instance_databases_governance(&mut tx, [dbname.trim()]) .await?; let mut logs = CustomInstanceDbLogs::default(); - let result = setup_custom_instance_pg_database_inner(authed, &db, &dbname, &mut logs).await; + let result = + setup_custom_instance_pg_database_inner(authed.clone(), &db, &dbname, &mut logs).await; let success = result.is_ok(); let error = result.err().map(|e| e.to_string()); let status = CustomInstanceDb { @@ -1990,6 +2044,20 @@ async fn setup_custom_instance_pg_database( .fetch_one(&mut *tx) .await?; tx.commit().await?; + audit_instance_action( + &db, + &authed, + "settings.setup_custom_instance_pg_database", + ActionKind::Create, + Some( + [ + ("dbname", dbname.as_str()), + ("success", if success { "true" } else { "false" }), + ] + .into(), + ), + ) + .await?; let status: CustomInstanceDb = serde_json::from_value(saved).map_err(to_anyhow)?; Ok(Json(status)) @@ -2168,6 +2236,14 @@ async fn drop_custom_instance_pg_database( require_super_admin(&db, &authed).await?; windmill_common::drop_custom_instance_database(&db, &dbname).await?; + audit_instance_action( + &db, + &authed, + "settings.drop_custom_instance_pg_database", + ActionKind::Delete, + Some([("dbname", dbname.as_str())].into()), + ) + .await?; Ok(format!("Database '{}' dropped successfully", dbname)) } @@ -2212,6 +2288,14 @@ pub async fn migrate_secrets_to_vault( let report = windmill_common::secret_backend::migrate_secrets_to_vault(&db, &settings).await?; + audit_instance_action( + &db, + &authed, + "settings.migrate_secrets_to_vault", + ActionKind::Update, + None, + ) + .await?; Ok(Json(report)) } @@ -2233,6 +2317,14 @@ pub async fn migrate_secrets_to_database( let report = windmill_common::secret_backend::migrate_secrets_to_database(&db, &settings).await?; + audit_instance_action( + &db, + &authed, + "settings.migrate_secrets_to_database", + ActionKind::Update, + None, + ) + .await?; Ok(Json(report)) } @@ -2266,6 +2358,14 @@ pub async fn migrate_secrets_to_azure_kv( let report = windmill_common::secret_backend::migrate_secrets_to_azure_kv(&db, &settings).await?; + audit_instance_action( + &db, + &authed, + "settings.migrate_secrets_to_azure_kv", + ActionKind::Update, + None, + ) + .await?; Ok(Json(report)) } @@ -2283,6 +2383,14 @@ pub async fn migrate_secrets_from_azure_kv( let report = windmill_common::secret_backend::migrate_secrets_from_azure_kv(&db, &settings).await?; + audit_instance_action( + &db, + &authed, + "settings.migrate_secrets_from_azure_kv", + ActionKind::Update, + None, + ) + .await?; Ok(Json(report)) } @@ -2307,6 +2415,14 @@ pub async fn migrate_secrets_to_aws_sm( ) -> JsonResult { require_super_admin(&db, &authed).await?; let report = windmill_common::secret_backend::migrate_secrets_to_aws_sm(&db, &settings).await?; + audit_instance_action( + &db, + &authed, + "settings.migrate_secrets_to_aws_sm", + ActionKind::Update, + None, + ) + .await?; Ok(Json(report)) } @@ -2320,6 +2436,14 @@ pub async fn migrate_secrets_from_aws_sm( require_super_admin(&db, &authed).await?; let report = windmill_common::secret_backend::migrate_secrets_from_aws_sm(&db, &settings).await?; + audit_instance_action( + &db, + &authed, + "settings.migrate_secrets_from_aws_sm", + ActionKind::Update, + None, + ) + .await?; Ok(Json(report)) } @@ -2342,95 +2466,6 @@ pub async fn get_jwks() -> JsonResult { Ok(Json(JwksResponse { keys: vec![] })) } -#[derive(serde::Deserialize, serde::Serialize)] -struct CachedResourceType { - #[allow(dead_code)] - id: i64, - name: String, - schema: Option, - #[allow(dead_code)] - app: String, - description: Option, - /// Doubly optional, and read through a wrapping deserializer: this struct also - /// decodes the on-disk cache, where an absent key means "written before the - /// column, leave the stored extension alone" and an explicit null means the hub - /// dropped it. Plain serde folds both into `None`. - #[serde( - default, - deserialize_with = "windmill_common::more_serde::double_option" - )] - format_extension: Option>, - /// Doubly optional like `format_extension`: no key leaves the stored name alone, an explicit - /// null (the hub naming nothing) clears it. - #[serde( - default, - deserialize_with = "windmill_common::more_serde::double_option" - )] - display_name: Option>, -} - -#[derive(serde::Deserialize)] -struct HubResourceTypeRaw { - id: i64, - name: String, - schema: Option, - app: String, - description: Option, - #[serde(default)] - format_extension: Option, - #[serde( - default, - deserialize_with = "windmill_common::more_serde::double_option" - )] - display_name: Option>, -} - -async fn fetch_resource_types_from_hub() -> error::Result> { - let response = HTTP_CLIENT - .get(format!( - "{}/resource_types/list", - windmill_common::DEFAULT_HUB_BASE_URL - )) - .header("Accept", "application/json") - .send() - .await - .map_err(|e| error::Error::InternalErr(format!("Failed to fetch from hub: {}", e)))?; - - if !response.status().is_success() { - return Err(error::Error::InternalErr(format!( - "Hub returned status {}", - response.status() - ))); - } - - let raw_types: Vec = response - .json() - .await - .map_err(|e| error::Error::InternalErr(format!("Failed to parse hub response: {}", e)))?; - - Ok(raw_types - .into_iter() - .filter_map(|rt| { - let schema = match rt.schema { - Some(s) => match serde_json::from_str(&s) { - Ok(v) => Some(v), - Err(_) => return None, - }, - None => None, - }; - Some(CachedResourceType { - id: rt.id, - name: rt.name, - schema, - app: rt.app, - description: rt.description, - format_extension: Some(rt.format_extension), - display_name: rt.display_name, - }) - }) - .collect()) -} - #[derive(serde::Deserialize)] struct SyncResourceTypesQuery { name: Option, @@ -2443,118 +2478,45 @@ async fn sync_cached_resource_types( ) -> error::Result { require_super_admin(&db, &authed).await?; - use windmill_common::worker::HUB_RT_CACHE_DIR; - let cache_path = format!("{}/resource_types.json", *HUB_RT_CACHE_DIR); + let outcome = hub_resource_types::sync(&db).await?; - // Manual sync is hub-first so it lands newly-published hub types on demand. The - // on-disk cache is only a fallback for when the hub is unreachable (airgapped - // installs / network error); refreshing it is left to the daily cache-rt cron and - // the startup sync in main.rs, which own the offline path. - let (resource_types, from_hub) = match fetch_resource_types_from_hub().await { - Ok(types) => { - tracing::info!("Fetched {} resource types live from the hub", types.len()); - (types, true) - } - Err(hub_err) => { - tracing::warn!( - "Live hub fetch failed ({hub_err}), falling back to on-disk cache at {cache_path}" - ); - match tokio::fs::read_to_string(&cache_path).await { - Ok(content) => { - let parsed = serde_json::from_str::>(&content) - .map_err(|e| { - error::Error::InternalErr(format!( - "Failed to parse cached resource types: {}", - e - )) - })?; - (parsed, false) - } - Err(_) => return Err(hub_err), - } - } - }; - - let mut synced_count = 0; - - for rt in &resource_types { - // A name too long for the column counts as absent, leaving the stored one alone: one bad - // entry must not fail the upsert and end the rest of the sync. - let display_name = match &rt.display_name { - Some(Some(name)) if name.chars().count() > 100 => None, - other => other.clone(), + if !outcome.changed.is_empty() { + let source = match outcome.source { + SyncSource::Hub => "hub", + SyncSource::ImageCache => "image_cache", }; - let exists: Option = sqlx::query_scalar!( - "SELECT EXISTS(SELECT 1 FROM resource_type WHERE workspace_id = 'admins' AND name = $1 AND schema IS NOT DISTINCT FROM $2 AND description IS NOT DISTINCT FROM $3 AND ($5 IS NOT TRUE OR format_extension IS NOT DISTINCT FROM $4) AND ($7 IS NOT TRUE OR display_name IS NOT DISTINCT FROM $6))", - &rt.name, - rt.schema.as_ref(), - rt.description.as_deref(), - rt.format_extension.clone().flatten(), - rt.format_extension.is_some(), - display_name.clone().flatten(), - display_name.is_some(), + let changed = outcome.changed.join(", "); + audit_log( + &db, + &authed, + "resource_types.sync", + ActionKind::Update, + "admins", + None, + Some([("source", source), ("changed", changed.as_str())].into()), ) - .fetch_one(&db) .await?; - - if exists.unwrap_or(false) { - continue; - } - - sqlx::query!( - // Whether the payload carried the key at all is what decides: present - // (even as null) is authoritative and may clear, absent means a cache - // written before the column and must leave the stored value alone. - "INSERT INTO resource_type (workspace_id, name, schema, description, format_extension, display_name, edited_at) - VALUES ('admins', $1, $2, $3, $4, $6, now()) - ON CONFLICT (workspace_id, name) DO UPDATE - SET schema = EXCLUDED.schema, description = EXCLUDED.description, - -- A fileset is a set of files, so it cannot also be one file. - -- Create and update reject the pair; this writer bypasses both, so - -- it declines the extension rather than persisting the forbidden - -- combination onto a same-named local fileset. - format_extension = CASE - WHEN resource_type.is_fileset THEN NULL - WHEN $5 THEN EXCLUDED.format_extension - ELSE resource_type.format_extension END, - display_name = CASE WHEN $7 THEN EXCLUDED.display_name ELSE resource_type.display_name END, - edited_at = now()", - &rt.name, - rt.schema.as_ref(), - rt.description.as_deref(), - rt.format_extension.clone().flatten(), - rt.format_extension.is_some(), - display_name.clone().flatten(), - display_name.is_some(), - ) - .execute(&db) - .await?; - - synced_count += 1; } // If a specific type was requested and is still absent after syncing, surface an // explicit not-found instead of a silent "Synced 0". Word it by source so the // cache-fallback path does not claim it checked the hub. if let Some(name) = name.as_deref() { - if !resource_types.iter().any(|rt| rt.name == name) { - let source = if from_hub { - "on the hub" - } else { - "in the cached resource types (hub unreachable)" + if !outcome.listed.iter().any(|listed| listed == name) { + let source = match outcome.source { + SyncSource::Hub => "on the hub", + SyncSource::ImageCache => "in the cached resource types (hub unavailable)", }; return Err(error::Error::NotFound(format!( - "resource type '{}' not found {}", - name, source + "resource type '{name}' not found {source}" ))); } } - Ok(format!( - "Synced {} resource types ({} unchanged)", - synced_count, - resource_types.len() - synced_count - )) + Ok(match outcome.source { + SyncSource::Hub => outcome.summary(), + SyncSource::ImageCache => format!("{}, the hub being unavailable", outcome.summary()), + }) } #[cfg(test)] diff --git a/backend/windmill-api-users/src/users.rs b/backend/windmill-api-users/src/users.rs index ac9defcd53..46bd6825e8 100644 --- a/backend/windmill-api-users/src/users.rs +++ b/backend/windmill-api-users/src/users.rs @@ -490,7 +490,7 @@ async fn list_user_usage( " SELECT usr.email, usage.executions FROM usr, LATERAL ( - SELECT COALESCE(SUM(c.duration_ms + 1000)/1000 , 0)::BIGINT executions + SELECT COALESCE(SUM(COALESCE((c.extras->>'wac_last_segment_ms')::bigint, c.duration_ms) + 1000)/1000 , 0)::BIGINT executions FROM v2_job_completed c JOIN v2_job j USING (id) WHERE j.workspace_id = $1 AND j.kind NOT IN ('flow', 'flowpreview', 'flownode', 'singlestepflow') diff --git a/backend/windmill-api-workers/src/lib.rs b/backend/windmill-api-workers/src/lib.rs index 0102f1b033..8ebee2a967 100644 --- a/backend/windmill-api-workers/src/lib.rs +++ b/backend/windmill-api-workers/src/lib.rs @@ -41,6 +41,7 @@ pub fn global_service() -> Router { .route("/queue_metrics", get(get_queue_metrics)) .route("/queue_metrics_series", get(get_queue_metrics_series)) .route("/queue_status", get(get_queue_status)) + .route("/runnable_stats", get(get_runnable_stats)) .route("/queue_counts", get(get_queue_counts)) .route("/queue_running_counts", get(get_queue_running_counts)) .route( @@ -86,6 +87,7 @@ struct WorkerPing { job_isolation: Option, #[serde(skip_serializing_if = "Option::is_none")] native_mode: Option, + draining: bool, } // #[derive(Serialize, Deserialize)] @@ -118,7 +120,7 @@ async fn list_worker_pings( WorkerPing, "SELECT worker, worker_instance, EXTRACT(EPOCH FROM (now() - ping_at))::integer as last_ping, started_at, ip, jobs_executed, CASE WHEN $4 IS TRUE THEN current_job_id ELSE NULL END as last_job_id, CASE WHEN $4 IS TRUE THEN current_job_workspace_id ELSE NULL END as last_job_workspace_id, - custom_tags, worker_group, wm_version, occupancy_rate, occupancy_rate_15s, occupancy_rate_5m, occupancy_rate_30m, memory, vcpus, memory_usage, wm_memory_usage, job_isolation, native_mode + custom_tags, worker_group, wm_version, occupancy_rate, occupancy_rate_15s, occupancy_rate_5m, occupancy_rate_30m, memory, vcpus, memory_usage, wm_memory_usage, job_isolation, native_mode, draining FROM worker_ping WHERE ($1::integer IS NULL AND ping_at > now() - interval '5 minute') OR (ping_at > now() - ($1 || ' seconds')::interval) ORDER BY ping_at desc LIMIT $2 OFFSET $3", @@ -298,6 +300,81 @@ async fn get_queue_metrics( Ok(Json(queue_metrics)) } +#[derive(Deserialize)] +struct RunnableStatsQuery { + window_secs: Option, + worker_group: Option, + workspace: Option, + order_by: Option, + limit: Option, +} + +#[derive(Serialize)] +struct RunnableStats { + workspace_id: String, + runnable_path: String, + worker_group: String, + job_count: i64, + total_duration_ms: i64, + max_memory_peak: i32, + sum_memory_peak: i64, + memory_sample_count: i64, + total_cpu_ms: i64, +} + +async fn get_runnable_stats( + authed: ApiAuthed, + Extension(db): Extension, + Query(query): Query, +) -> JsonResult> { + require_devops_role(&db, &authed).await?; + + let window = query.window_secs.unwrap_or(24 * 3600).clamp( + 3600, + windmill_common::runnable_job_stats::RETENTION_DAYS * 24 * 3600, + ); + // Rows are hourly buckets keyed by their start, so the bucket the window begins in + // is included whole. + let from_hour = (chrono::Utc::now().timestamp() - window) / 3600 * 3600; + let order_by = match query.order_by.as_deref() { + Some("max_memory") => "max_memory", + Some("cpu") => "cpu", + _ => "total_duration", + }; + + let rows = sqlx::query_as!( + RunnableStats, + r#" + SELECT workspace_id, runnable_path, worker_group, + SUM(job_count)::bigint AS "job_count!", + SUM(total_duration_ms)::bigint AS "total_duration_ms!", + MAX(max_memory_peak) AS "max_memory_peak!", + SUM(sum_memory_peak)::bigint AS "sum_memory_peak!", + SUM(memory_sample_count)::bigint AS "memory_sample_count!", + SUM(total_cpu_ms)::bigint AS "total_cpu_ms!" + FROM runnable_job_stats + WHERE hour >= $1 + AND ($2::text IS NULL OR worker_group = $2) + AND ($3::text IS NULL OR workspace_id = $3) + GROUP BY workspace_id, runnable_path, worker_group + ORDER BY CASE $4::text + WHEN 'max_memory' THEN MAX(max_memory_peak)::numeric + WHEN 'cpu' THEN SUM(total_cpu_ms) + ELSE SUM(total_duration_ms) + END DESC, workspace_id, runnable_path, worker_group + LIMIT $5 + "#, + from_hour, + query.worker_group, + query.workspace, + order_by, + query.limit.unwrap_or(100).clamp(1, 1000), + ) + .fetch_all(&db) + .await?; + Ok(Json(rows)) +} + #[derive(Deserialize)] struct QueueMetricsSeriesQuery { window_secs: Option, diff --git a/backend/windmill-api-workspaces/src/datatable_acl.rs b/backend/windmill-api-workspaces/src/datatable_acl.rs index ebab6214ce..efaa712481 100644 --- a/backend/windmill-api-workspaces/src/datatable_acl.rs +++ b/backend/windmill-api-workspaces/src/datatable_acl.rs @@ -683,6 +683,9 @@ pub(crate) struct CatalogFacts { pub(crate) former_owner: Option, /// For a revoke: each grant it takes back. pub(crate) revoked_grants: Vec, + /// For a grant on what a schema holds: the role cannot reach the schema yet, so the grant + /// would give it nothing it can use. + pub(crate) lacks_schema_usage: bool, } /// A grant a revoke takes back, as the catalog records it: what `source` gave on `object` (the @@ -1526,7 +1529,24 @@ async fn build_plan( ))); } } - let facts = CatalogFacts { other_pg_roles, existing_objects, former_owner, revoked_grants }; + let lacks_schema_usage = match (&change, target) { + (AclChange::Grant { scope, .. }, AclTarget::Schema { schema }) + if !matches!(scope, GrantScope::Target) => + { + !has_schema_usage(client, schema, &pg_role).await? + } + (AclChange::Grant { .. }, AclTarget::Table { schema, .. }) => { + !has_schema_usage(client, schema, &pg_role).await? + } + _ => false, + }; + let facts = CatalogFacts { + other_pg_roles, + existing_objects, + former_owner, + revoked_grants, + lacks_schema_usage, + }; let mut plan = crate::datatable_acl_oss::plan_statements(target, &change, dbname, &pg_role, &facts)?; if matches!(change, AclChange::SetOwner { .. }) { @@ -1540,6 +1560,27 @@ async fn build_plan( Ok(plan) } +/// Whether `pg_role` may look into `schema`, directly or through a role it inherits from. +async fn has_schema_usage( + client: &tokio_postgres::Client, + schema: &str, + pg_role: &str, +) -> Result { + Ok(client + .query_one( + "SELECT has_schema_privilege($1::name, $2::text, 'USAGE')", + &[&pg_role, &schema], + ) + .await + .map_err(|e| { + Error::internal_err(format!( + "Failed to read the role's privileges on schema {schema}: {}", + pg_error_message(&e) + )) + })? + .get(0)) +} + /// Postgres only hands an object to a role that could have created it: a table to one with /// `CREATE` on its schema, a schema to one with `CREATE` on the database. async fn missing_owner_privilege( diff --git a/backend/windmill-api-workspaces/src/datatable_migrations.rs b/backend/windmill-api-workspaces/src/datatable_migrations.rs index f762fda175..e63e69d4d5 100644 --- a/backend/windmill-api-workspaces/src/datatable_migrations.rs +++ b/backend/windmill-api-workspaces/src/datatable_migrations.rs @@ -17,6 +17,7 @@ use crate::workspaces::{ use axum::{ extract::{Extension, Path, Query}, + http::StatusCode, routing::{delete, get, post}, Json, Router, }; @@ -235,6 +236,7 @@ async fn run_datatable_migration_job( args.insert("database".to_string(), database_arg.clone()); let push_args = PushArgs { extra: None, args: &args }; + let scope_ceiling = windmill_api_auth::caller_scope_ceiling(db, authed).await?; let (uuid, mut tx) = push( db, PushIsolationLevel::Isolated(user_db.clone(), authed.clone().into()), @@ -278,6 +280,7 @@ async fn run_datatable_migration_job( None, None, None, + scope_ceiling.as_deref(), ) .await?; @@ -401,6 +404,12 @@ async fn connection_identity(client: &tokio_postgres::Client) -> Option<(String, }) } +const MIGRATION_LOCK_KEY: &str = "hashtext('windmill_datatable_migrations')::int8"; + +/// How long a run, rollback or edit waits for the one ahead of it. A migration may run for +/// longer than any request should wait, so past this the caller is told to retry. +const MIGRATION_LOCK_WAIT_SECS: u64 = 30; + /// Open a connection to a data table's own database and hold the session-level /// advisory lock that serializes migration runs/rollbacks. The lock is released /// when the returned client is dropped, so callers must keep it in scope for the @@ -426,18 +435,85 @@ async fn lock_datatable_migration_runs( tracing::error!("Datatable connection error: {}", e); } }); - client - .batch_execute("SELECT pg_advisory_lock(hashtext('windmill_datatable_migrations')::int8)") - .await - .map_err(|e| { - Error::internal_err(format!( - "Failed to acquire migration lock: {}", - pg_error_message(&e) - )) - })?; + hold_migration_lock(&client, datatable_name, MIGRATION_LOCK_WAIT_SECS).await?; Ok(client) } +async fn hold_migration_lock( + client: &tokio_postgres::Client, + datatable_name: &str, + wait_secs: u64, +) -> Result<()> { + // A session whose client vanished without closing the socket otherwise keeps the lock for + // the server's keepalive default, two hours on Linux. `SET` rather than startup `options`, + // which PgBouncer rejects. Behind a pooler they only cover the pooler's own connection to + // the server: there, as on a server refusing them, the bounded wait below is the recovery. + if let Err(e) = client + .batch_execute( + "SET tcp_keepalives_idle = 60; SET tcp_keepalives_interval = 10; \ + SET tcp_keepalives_count = 6", + ) + .await + { + tracing::debug!( + "Could not set server keepalives on the migration lock connection: {}", + pg_error_message(&e) + ); + } + let locked = client + .batch_execute(&format!( + "SET lock_timeout = '{wait_secs}s'; \ + SELECT pg_advisory_lock({MIGRATION_LOCK_KEY}); \ + RESET lock_timeout" + )) + .await; + match locked { + Ok(()) => Ok(()), + Err(e) if e.code() == Some(&SqlState::LOCK_NOT_AVAILABLE) => { + // The key as `pg_locks` splits it: high half in `classid`, low half in `objid`. + let holder = client + .query_opt( + &format!( + "SELECT pid FROM pg_locks + WHERE locktype = 'advisory' AND granted AND objsubid = 1 + AND database = (SELECT oid FROM pg_database WHERE datname = current_database()) + AND ((classid::bigint << 32) | objid::bigint) = {MIGRATION_LOCK_KEY}" + ), + &[], + ) + .await + .ok() + .flatten() + .map(|row| row.get::<_, i32>(0)); + let held_by = match holder { + Some(pid) => format!( + " The lock is held by backend pid {pid}: if nothing is running, a lost \ + connection left that session behind, and \ + `SELECT pg_terminate_backend({pid})` on the data table's database releases it." + ), + None => String::new(), + }; + Err(Error::Generic( + StatusCode::CONFLICT, + format!( + "Another migration run, rollback or edit on the database of data table \ + '{datatable_name}' was still in progress after {wait_secs}s. \ + Retry once it is done.{held_by}" + ), + )) + } + Err(e) => Err(Error::internal_err(format!( + "Failed to acquire migration lock: {}", + pg_error_message(&e) + ))), + } +} + +/// Whether `e` is the lock wait running out, as opposed to the database being unreachable. +fn is_migration_lock_busy(e: &Error) -> bool { + matches!(e, Error::Generic(StatusCode::CONFLICT, _)) +} + /// Read the versions recorded as applied in a data table's `_wm_migrations`, /// scoped to that data table, using an existing connection. An absent table /// (`42P01`) means nothing has been migrated yet. @@ -1358,7 +1434,13 @@ async fn delete_datatable_migration( }; let lock_client = lock_datatable_migration_runs(&db, &w_id, &datatable_name) .await - .map_err(unreachable)?; + .map_err(|e| { + if is_migration_lock_busy(&e) { + e + } else { + unreachable(e) + } + })?; let applied = read_applied_versions_on_client(&lock_client, &datatable_name) .await .map_err(unreachable)?; @@ -1533,7 +1615,13 @@ async fn upsert_datatable_migration( }; let client = lock_datatable_migration_runs(&db, &w_id, &datatable_name) .await - .map_err(unreachable)?; + .map_err(|e| { + if is_migration_lock_busy(&e) { + e + } else { + unreachable(e) + } + })?; let applied = read_applied_versions_on_client(&client, &datatable_name) .await .map_err(unreachable)?; @@ -2041,6 +2129,32 @@ pub(crate) async fn clone_datatable_migrations( mod tests { use super::*; + // The holder is found by reassembling the key from the two halves `pg_locks` shows. + #[sqlx::test(migrations = false)] + async fn a_held_migration_lock_is_reported_with_its_holder(pool: DB) { + let mut config: tokio_postgres::Config = + std::env::var("DATABASE_URL").unwrap().parse().unwrap(); + config.dbname(pool.connect_options().get_database().unwrap()); + let (holder, connection) = config.connect(tokio_postgres::NoTls).await.unwrap(); + tokio::spawn(connection); + let (waiter, connection) = config.connect(tokio_postgres::NoTls).await.unwrap(); + tokio::spawn(connection); + hold_migration_lock(&holder, "main", 1).await.unwrap(); + let holder_pid = holder + .query_one("SELECT pg_backend_pid()", &[]) + .await + .unwrap() + .get::<_, i32>(0); + + let busy = hold_migration_lock(&waiter, "main", 1).await.unwrap_err(); + + assert!(is_migration_lock_busy(&busy), "{busy:?}"); + assert!( + busy.to_string().contains(&format!("pid {holder_pid}:")), + "{busy}" + ); + } + #[test] fn validate_migration_name_accepts_safe_names() { for name in ["initial", "add_index_to_customers", "fix-bug_2", "ABC123"] { diff --git a/backend/windmill-api-workspaces/src/remote_deploy.rs b/backend/windmill-api-workspaces/src/remote_deploy.rs index 3e701cbf14..a0f922a607 100644 --- a/backend/windmill-api-workspaces/src/remote_deploy.rs +++ b/backend/windmill-api-workspaces/src/remote_deploy.rs @@ -25,11 +25,11 @@ use axum::{ }; use chrono::{DateTime, Utc}; use lazy_static::lazy_static; -use magic_crypt::MagicCrypt256; use regex::Regex; use serde::{Deserialize, Serialize}; use windmill_api_auth::{is_effectively_unscoped, ApiAuthed, Tokened}; use windmill_audit::{audit_oss::audit_log, ActionKind}; +use windmill_common::variables::WorkspaceCrypt; use windmill_common::{ auth::hash_token, error::{error_source_chain, Error, Result}, @@ -676,8 +676,8 @@ async fn proxy( pub(crate) async fn reencrypt_tokens( conn: &mut sqlx::PgConnection, w_id: &str, - old: &MagicCrypt256, - new: &MagicCrypt256, + old: &WorkspaceCrypt, + new: &WorkspaceCrypt, ) -> Result<()> { let rows = sqlx::query!( "SELECT email, token FROM remote_deploy_token diff --git a/backend/windmill-api-workspaces/src/workspaces.rs b/backend/windmill-api-workspaces/src/workspaces.rs index 81fead981b..3bfcef5c96 100644 --- a/backend/windmill-api-workspaces/src/workspaces.rs +++ b/backend/windmill-api-workspaces/src/workspaces.rs @@ -744,13 +744,15 @@ async fn is_premium( #[derive(Serialize)] struct BillableSeatsResponse { - /// Both omitted when the seats counted are another workspace's: a fork member need not be a + /// All three omitted when the seats counted are another workspace's: a fork member need not be a /// member of the billing root, so the root's headcount is not theirs to read. The total is, /// since it is the divisor of the quota their own executions draw on. #[serde(skip_serializing_if = "Option::is_none")] developers: Option, #[serde(skip_serializing_if = "Option::is_none")] operators: Option, + #[serde(skip_serializing_if = "Option::is_none")] + service_accounts: Option, seats: i64, } @@ -780,6 +782,7 @@ async fn get_billable_seats( Ok(Json(BillableSeatsResponse { developers: own.then_some(counted.developers), operators: own.then_some(counted.operators), + service_accounts: own.then_some(counted.service_accounts), seats: counted.seats, })) } @@ -6314,8 +6317,8 @@ async fn set_encryption_key( async fn reencrypt_git_sync_secrets( conn: &mut sqlx::PgConnection, w_id: &str, - old: &magic_crypt::MagicCrypt256, - new: &magic_crypt::MagicCrypt256, + old: &windmill_common::variables::WorkspaceCrypt, + new: &windmill_common::variables::WorkspaceCrypt, ) -> Result<()> { let Some((mut credentials, mut git_sync)) = sqlx::query_as::<_, (serde_json::Value, Option)>( @@ -7637,7 +7640,7 @@ async fn clone_scripts( dedicated_worker, ws_error_handler_muted, priority, timeout, delete_after_use, delete_after_secs, restart_unless_cancelled, concurrency_key, visible_to_runner_only, auto_kind, codebase, has_preprocessor, - on_behalf_of, on_behalf_of_email, assets, modules + on_behalf_of, on_behalf_of_email, assets, modules, job_token_scopes ) SELECT $1, hash, path, parent_hashes, summary, description, content, @@ -7669,7 +7672,7 @@ async fn clone_scripts( UNION ALL SELECT 1 FROM password p WHERE p.email = on_behalf_of AND p.super_admin)) - END, on_behalf_of_email, assets, modules + END, on_behalf_of_email, assets, modules, job_token_scopes FROM script WHERE workspace_id = $2"#, target_workspace_id, @@ -8027,7 +8030,8 @@ async fn clone_flows( workspace_id, path, summary, description, value, edited_by, edited_at, archived, schema, extra_perms, dependency_job, tag, ws_error_handler_muted, dedicated_worker, timeout, visible_to_runner_only, - concurrency_key, versions, on_behalf_of, on_behalf_of_email, lock_error_logs + concurrency_key, versions, on_behalf_of, on_behalf_of_email, lock_error_logs, + job_token_scopes ) SELECT $2, path, summary, description, value, edited_by, edited_at, archived, schema, extra_perms, NULL, tag, @@ -8053,7 +8057,7 @@ async fn clone_flows( UNION ALL SELECT 1 FROM password p WHERE p.email = on_behalf_of AND p.super_admin)) - END, on_behalf_of_email, lock_error_logs + END, on_behalf_of_email, lock_error_logs, job_token_scopes FROM flow WHERE workspace_id = $1", source_workspace_id, @@ -10509,7 +10513,7 @@ async fn edit_workspace( pub(crate) async fn archive_workspace_impl( db: &DB, w_id: &str, - username: &str, + authed: &ApiAuthed, // When archiving a dev workspace, its parent prod. The pairing teardown (clear is_dev + drop the // prod's lock) is folded into the same transaction as `deleted = true` so it's atomic with the // archive — a later failure can't strand a half-archived dev that's still flagged/locked. @@ -10623,7 +10627,7 @@ pub(crate) async fn archive_workspace_impl( let axum::Json(canceled_jobs) = windmill_api_jobs::cancel_jobs( jobs_to_cancel, db, - username, + authed, w_id, false, // force_cancel ) @@ -10683,7 +10687,7 @@ async fn archive_workspace( // dev workspace, which only applies to a workspace that is itself a dev (`dev_lock_parent`): a // root archived out from under its dev is the pre-existing shape and not this pairing's to police. let (schedules_count, canceled_count, deleted_tokens_count) = - archive_workspace_impl(&db, &w_id, &authed.username, dev_lock_parent.as_deref()).await?; + archive_workspace_impl(&db, &w_id, &authed, dev_lock_parent.as_deref()).await?; // Audit log let mut tx = db.begin().await?; @@ -11604,10 +11608,13 @@ struct ChangeOperatorSettings { /// withdraw or grant the right on every push. /// /// `builder_flows` lets every operator of this workspace compose flows out of already-deployed - /// runnables, and makes each of them consume a full author seat instead of half of one. + /// runnables, `builder_apps` full-code apps. Either makes each operator consume a full author + /// seat instead of half of one. #[serde(default, skip_serializing_if = "Option::is_none")] builder_flows: Option, #[serde(default, skip_serializing_if = "Option::is_none")] + builder_apps: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] manage_schedules: Option, #[serde(default, skip_serializing_if = "Option::is_none")] manage_triggers: Option, @@ -11622,9 +11629,10 @@ async fn update_operator_settings( require_admin(authed.is_admin, &authed.username)?; // Every operator of the workspace turns into a full seat, which an offline license may not - // cover. It is a no-op delta when the right is already on. + // cover. Either right costs the same seat, so this prices "any", and it is a no-op delta when + // one of them is already on. #[cfg(feature = "enterprise")] - if settings.builder_flows == Some(true) { + if settings.builder_flows == Some(true) || settings.builder_apps == Some(true) { if let Some(msg) = windmill_common::ee_oss::check_seat_cap_for_operator_builder(&db, &w_id).await? { @@ -14046,6 +14054,12 @@ const TRIGGER_COMPARE_IGNORE: &[&str] = &[ "last_server_ping", "extra_perms", "permissioned_as", + // Schedule counters the scheduler bumps as occurrences are missed. A fork's + // schedules are disabled so theirs stay at the defaults, and no deploy carries + // them across: compared, a parent schedule that ever ran late differs forever. + "late_run_streak", + "missed_occurrences", + "last_missed_at", // Server-managed fields that the merge feature treats as workspace-local // (regenerated by the deploy handler): GCP `subscription_id` is rewritten // to `windmill__` in `CreateUpdate` mode, and Azure's diff --git a/backend/windmill-api-workspaces/src/workspaces_extra.rs b/backend/windmill-api-workspaces/src/workspaces_extra.rs index 8ca98f66d1..2f0d4deb13 100644 --- a/backend/windmill-api-workspaces/src/workspaces_extra.rs +++ b/backend/windmill-api-workspaces/src/workspaces_extra.rs @@ -408,8 +408,8 @@ pub(crate) async fn change_workspace_id( info!("Duplicating flow table rows"); sqlx::query!( "INSERT INTO flow - (workspace_id, path, summary, description, archived, extra_perms, dependency_job, tag, ws_error_handler_muted, dedicated_worker, timeout, visible_to_runner_only, on_behalf_of, on_behalf_of_email, concurrency_key, versions, value, schema, edited_by, edited_at, lock_error_logs) - SELECT $1, path, summary, description, archived, extra_perms, dependency_job, tag, ws_error_handler_muted, dedicated_worker, timeout, visible_to_runner_only, on_behalf_of, on_behalf_of_email, concurrency_key, versions, value, schema, edited_by, edited_at, lock_error_logs + (workspace_id, path, summary, description, archived, extra_perms, dependency_job, tag, ws_error_handler_muted, dedicated_worker, timeout, visible_to_runner_only, on_behalf_of, on_behalf_of_email, concurrency_key, versions, value, schema, edited_by, edited_at, lock_error_logs, job_token_scopes) + SELECT $1, path, summary, description, archived, extra_perms, dependency_job, tag, ws_error_handler_muted, dedicated_worker, timeout, visible_to_runner_only, on_behalf_of, on_behalf_of_email, concurrency_key, versions, value, schema, edited_by, edited_at, lock_error_logs, job_token_scopes FROM flow WHERE workspace_id = $2", &rw.new_id, &old_id @@ -944,7 +944,7 @@ pub(crate) async fn change_workspace_id( // Note: schedules were already moved to new workspace, so this will find 0 schedules info!("Archiving old workspace"); let (_schedules_count, canceled_count, _deleted_tokens_count) = - archive_workspace_impl(&db, &old_id, &authed.username, None).await?; + archive_workspace_impl(&db, &old_id, &authed, None).await?; // The old id stays live between the commit above and the archive, and a fork copy created for // it in that window registers under it. Creation checks the workspace is live under the fork @@ -1299,6 +1299,13 @@ pub(crate) async fn delete_workspace( .execute(&mut *tx) .await?; + sqlx::query!( + "DELETE FROM runnable_job_stats WHERE workspace_id = $1", + &w_id + ) + .execute(&mut *tx) + .await?; + // If this workspace is itself a dev workspace, deleting it dissolves the pairing, so also drop // the parent prod's reserved dev_workspace_lock (mirrors detach_dev_workspace) — otherwise prod // stays locked against direct deploy/forking with no dev workspace left to make changes in. @@ -1670,13 +1677,12 @@ pub async fn drop_forked_datatable_databases( match parent_pg.connect(Some(&db)).await { Ok((client, connection)) => { let join_handle = tokio::spawn(async move { connection.await }); - if let Err(e) = client - .execute(&format!("DROP DATABASE \"{}\"", db_to_drop), &[]) - .await - { + if let Err(e) = drop_database_with_sessions(&client, db_to_drop).await { errors.push(format!( "Could not drop database '{}' for datatable://{}: {}", - db_to_drop, dt_name, e + db_to_drop, + dt_name, + windmill_common::error::pg_error_message(&e) )); } drop(client); @@ -1695,6 +1701,44 @@ pub async fn drop_forked_datatable_databases( Ok(Json(errors)) } +/// Drops `dbname`, whose name the caller validated, along with the sessions still connected to +/// it: a worker keeps an idle connection to a database it ran a job on, and a plain +/// `DROP DATABASE` fails for as long as one exists. +async fn drop_database_with_sessions( + client: &tokio_postgres::Client, + dbname: &str, +) -> std::result::Result<(), tokio_postgres::Error> { + let version = client + .query_one("SELECT current_setting('server_version_num')::int", &[]) + .await? + .get::<_, i32>(0); + if version >= 130000 { + client + .execute(&format!("DROP DATABASE \"{dbname}\" WITH (FORCE)"), &[]) + .await?; + return Ok(()); + } + // No `FORCE` before 13. The drop below waits a few seconds for the sessions to exit, and + // reports the ones this role was not allowed to terminate. + if let Err(e) = client + .execute( + "SELECT pg_terminate_backend(pid) FROM pg_stat_activity + WHERE datname = $1 AND pid <> pg_backend_pid()", + &[&dbname], + ) + .await + { + tracing::warn!( + "Failed to terminate connections to '{dbname}': {}", + windmill_common::error::pg_error_message(&e) + ); + } + client + .execute(&format!("DROP DATABASE \"{dbname}\""), &[]) + .await?; + Ok(()) +} + /// Drop this fork workspace's ducklake namespaces: the `wm_fork_*` metadata schema in each /// lake's catalog database, plus (best effort) the fork's `__wm_forks//…` data files in /// the workspace storage. Driven by the `fork_ducklake_namespace` registry written at first @@ -2265,3 +2309,47 @@ async fn workspace_is_fork(db: &DB, w_id: &str) -> Result { .await? .unwrap_or(false)) } + +#[cfg(test)] +mod tests { + use super::drop_database_with_sessions; + + #[sqlx::test(migrations = false)] + async fn a_database_is_dropped_under_a_connected_session(pool: sqlx::PgPool) { + let mut config: tokio_postgres::Config = + std::env::var("DATABASE_URL").unwrap().parse().unwrap(); + let test_db = pool.connect_options().get_database().unwrap().to_string(); + let (client, connection) = config + .dbname(&test_db) + .connect(tokio_postgres::NoTls) + .await + .unwrap(); + tokio::spawn(connection); + // The test database's name already fills an identifier. + let held_db = format!("held_{}", &test_db[test_db.len().saturating_sub(40)..]); + client + .execute(&format!("CREATE DATABASE \"{held_db}\""), &[]) + .await + .unwrap(); + let (_session, connection) = config + .dbname(&held_db) + .connect(tokio_postgres::NoTls) + .await + .unwrap(); + tokio::spawn(connection); + + drop_database_with_sessions(&client, &held_db) + .await + .unwrap(); + + let left = client + .query_one( + "SELECT count(*) FROM pg_database WHERE datname = $1", + &[&held_db], + ) + .await + .unwrap() + .get::<_, i64>(0); + assert_eq!(left, 0); + } +} diff --git a/backend/windmill-api/Cargo.toml b/backend/windmill-api/Cargo.toml index ad7f12453c..b3aea64f72 100644 --- a/backend/windmill-api/Cargo.toml +++ b/backend/windmill-api/Cargo.toml @@ -20,7 +20,7 @@ benchmark = [] embedding = ["windmill-api-embeddings/embedding"] parquet = ["dep:datafusion", "windmill-common/parquet", "windmill-object-store/parquet", "windmill-worker?/parquet", "windmill-indexer?/parquet", "windmill-api-users/parquet", "windmill-api-settings/parquet", "windmill-api-workspaces/parquet", "windmill-api-npm-proxy/parquet", "dep:aws-sigv4", "dep:aws-sdk-config", "dep:quick-xml"] prometheus = ["windmill-common/prometheus", "windmill-queue/prometheus", "dep:prometheus", "windmill-worker?/prometheus", "windmill-api-scripts/prometheus"] -openidconnect = ["dep:openidconnect", "windmill-common/openidconnect", "windmill-store/openidconnect"] +openidconnect = ["dep:openidconnect", "dep:rsa", "dep:p256", "dep:tokio-rustls", "windmill-common/openidconnect", "windmill-store/openidconnect"] # The service log search handler reads the columnar store, so tantivy alone is # not enough for this crate to build on its own. tantivy = ["dep:windmill-indexer", "parquet"] @@ -141,6 +141,8 @@ samael = { workspace = true, optional = true } libxml = { workspace = true, optional = true } async-recursion.workspace = true rsa = { workspace = true, optional = true} +p256 = { workspace = true, optional = true } +tokio-rustls = { workspace = true, optional = true } uuid.workspace = true datafusion = { workspace = true, optional = true} openidconnect = { workspace = true, optional = true} @@ -189,4 +191,7 @@ flate2.workspace = true strum = { workspace = true, optional = true } dashmap.workspace = true +[dev-dependencies] +rcgen.workspace = true + [build-dependencies] diff --git a/backend/windmill-api/openapi-deref.json b/backend/windmill-api/openapi-deref.json index 3cba2b7da6..4016222d30 100644 --- a/backend/windmill-api/openapi-deref.json +++ b/backend/windmill-api/openapi-deref.json @@ -5424,6 +5424,10 @@ "is_past_due": { "type": "boolean" }, + "is_canceled": { + "type": "boolean", + "description": "the team plan was canceled this month; it stays premium until the month ends, capped at max_tolerated_executions" + }, "max_tolerated_executions": { "type": "number" } @@ -34989,7 +34993,7 @@ }, "exponential": { "type": "object", - "description": "Retry with exponential backoff (delay doubles each time)", + "description": "Retry with exponential backoff: the n-th retry waits multiplier × seconds^n", "properties": { "attempts": { "type": "integer", @@ -35002,7 +35006,7 @@ "seconds": { "type": "integer", "minimum": 1, - "description": "Initial delay in seconds" + "description": "Base of the exponential (seconds); the n-th retry waits multiplier × seconds^n, where n counts constant retries too" }, "random_factor": { "type": "integer", @@ -47926,7 +47930,7 @@ }, "exponential": { "type": "object", - "description": "Retry with exponential backoff (delay doubles each time)", + "description": "Retry with exponential backoff: the n-th retry waits multiplier × seconds^n", "properties": { "attempts": { "type": "integer", @@ -47939,7 +47943,7 @@ "seconds": { "type": "integer", "minimum": 1, - "description": "Initial delay in seconds" + "description": "Base of the exponential (seconds); the n-th retry waits multiplier × seconds^n, where n counts constant retries too" }, "random_factor": { "type": "integer", diff --git a/backend/windmill-api/openapi-deref.yaml b/backend/windmill-api/openapi-deref.yaml index d5a14a717d..d541144231 100644 --- a/backend/windmill-api/openapi-deref.yaml +++ b/backend/windmill-api/openapi-deref.yaml @@ -5286,6 +5286,9 @@ paths: type: string is_past_due: type: boolean + is_canceled: + type: boolean + description: the team plan was canceled this month; it stays premium until the month ends, capped at max_tolerated_executions max_tolerated_executions: type: number required: @@ -11722,9 +11725,7 @@ paths: description: Seconds to wait between retries exponential: type: object - description: >- - Retry with exponential backoff (delay - doubles each time) + description: 'Retry with exponential backoff: the n-th retry waits multiplier × seconds^n' properties: attempts: type: integer @@ -11735,7 +11736,7 @@ paths: seconds: type: integer minimum: 1 - description: Initial delay in seconds + description: Base of the exponential (seconds); the n-th retry waits multiplier × seconds^n, where n counts constant retries too random_factor: type: integer minimum: 0 @@ -23269,7 +23270,7 @@ paths: description: Seconds to wait between retries exponential: type: object - description: Retry with exponential backoff (delay doubles each time) + description: 'Retry with exponential backoff: the n-th retry waits multiplier × seconds^n' properties: attempts: type: integer @@ -23280,7 +23281,7 @@ paths: seconds: type: integer minimum: 1 - description: Initial delay in seconds + description: Base of the exponential (seconds); the n-th retry waits multiplier × seconds^n, where n counts constant retries too random_factor: type: integer minimum: 0 diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index d9aeb63707..433d7c00be 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -1,7 +1,7 @@ openapi: "3.0.3" info: - version: 1.821.0 + version: 1.830.0 title: Windmill API contact: @@ -1765,6 +1765,12 @@ paths: type: string cluster: $ref: "#/components/schemas/DatatableRoleCluster" + take_over: + type: boolean + description: > + manage a Postgres role of that name that already exists on the cluster, + replacing its password, instead of refusing it. A role with cluster-wide + attributes or memberships is refused either way responses: "200": description: the created role @@ -1772,6 +1778,8 @@ paths: application/json: schema: $ref: "#/components/schemas/InstanceDatatableRole" + "409": + description: a Postgres role of that name already exists on the cluster and take_over was not set /settings/datatable_roles/{id}: post: @@ -1911,7 +1919,7 @@ paths: /settings/instance_ui: get: - summary: get the instance-wide announcement banner and accent color + summary: get the instance-wide announcement banner, accent color and API base url description: | Readable by any authenticated user; every page load reads it. Each value is the raw stored setting, or null when unset. @@ -1920,7 +1928,7 @@ paths: - setting responses: "200": - description: the instance_banner and accent_color settings + description: the instance_banner, accent_color and api_base_url settings content: application/json: schema: @@ -1928,6 +1936,7 @@ paths: properties: instance_banner: {} accent_color: {} + api_base_url: {} /settings_u/ruff_config: get: @@ -4492,6 +4501,12 @@ paths: description: >- Omitted when the seats counted are another workspace's, as they are for a fork resolving to its billing root. + service_accounts: + type: integer + description: >- + Enabled service accounts, half a seat each like operators. Omitted when the + seats counted are another workspace's, as they are for a fork resolving to + its billing root. seats: type: integer required: @@ -4529,6 +4544,9 @@ paths: type: string is_past_due: type: boolean + is_canceled: + type: boolean + description: the team plan was canceled this month; it stays premium until the month ends, capped at max_tolerated_executions max_tolerated_executions: type: number required: @@ -5550,6 +5568,9 @@ paths: type: boolean default_role: type: string + description: >- + The id of one of `roles`, `admin` (the default), or `$none` for no default + role: a connection that names no role then holds its viewer's roles only. roles: type: array items: @@ -7418,6 +7439,72 @@ paths: items: $ref: "#/components/schemas/TruncatedToken" + /oidc/signing_status: + get: + summary: what signs OIDC tokens and the keys the JWKS serves (ee only, superadmin) + operationId: getOidcSigningStatus + tags: + - oidc + responses: + "200": + description: OIDC signing status + content: + application/json: + schema: + type: object + required: + - mode + - keys + properties: + mode: + type: string + enum: [database, file, external] + keys: + type: array + items: + type: object + required: + - alg + - state + - source + properties: + kid: + type: string + alg: + type: string + state: + type: string + enum: [pending, active, retired, published] + source: + type: string + enum: [signer, database, file, publish_dir] + signer: + type: object + description: set when an external signer is configured + required: + - breaker + - max_rps + - current_rps + properties: + last_refresh_success: + type: string + format: date-time + last_refresh_error: + type: string + breaker: + type: string + enum: [closed, open, half_open] + max_rps: + type: number + current_rps: + type: number + keys_from_store: + type: boolean + description: the keys served are those stored by the last accepted listing, the signer has not answered this server yet + tls_files_error: + type: string + description: why the mTLS files on disk are not the ones in use + /w/{workspace}/oidc/token/{audience}: post: summary: get OIDC token (ee only) @@ -10907,6 +10994,37 @@ paths: description: "`saved` only, upsert or delete: where the write landed. Differs from the URL path when the item had moved away from it; the editor follows it there. Absent when a delete found nothing to remove and the caller cannot read the path it moved to." required: [status, current_timestamp] + /w/{workspace}/drafts/delete_for_user/{kind}/{path}: + post: + summary: delete another user's draft, or every draft at a path (admin only) + description: Delete the draft a given user holds at a path, or, without `username`, every draft at that path (the legacy ownerless one and the caller's own included). Workspace admins / superadmins only, and only for script, flow, app and raw_app drafts. Unlike a discard of one's own draft through `drafts/update`, this returns 404 when nothing was deleted. + operationId: deleteDraftForUser + tags: + - draft + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: kind + in: path + required: true + schema: + $ref: "#/components/schemas/UserDraftItemKind" + - $ref: "#/components/parameters/ScriptPath" + - name: username + in: query + required: false + description: Workspace username of the draft owner. Omit to delete every user's draft at the path. + schema: + type: string + responses: + "200": + description: draft(s) deleted + content: + text/plain: + schema: + type: string + "404": + description: no matching draft at that path + /w/{workspace}/drafts/move/{kind}/{path}: post: summary: move the current user's draft-only item to another path @@ -11995,6 +12113,7 @@ paths: - $ref: "#/components/parameters/WorkerTag" - $ref: "#/components/parameters/CacheTtl" - $ref: "#/components/parameters/NewJobId" + - $ref: "#/components/parameters/RunRetry" - name: invisible_to_owner description: make the run invisible to the the script owner (default false) in: query @@ -15999,6 +16118,7 @@ paths: - $ref: "#/components/parameters/WorkerTag" - $ref: "#/components/parameters/CacheTtl" - $ref: "#/components/parameters/NewJobId" + - $ref: "#/components/parameters/RunRetry" - $ref: "#/components/parameters/IncludeHeader" - name: invisible_to_owner description: make the run invisible to the the script owner (default false) @@ -16527,6 +16647,31 @@ paths: required: - database_length + /w/{workspace}/jobs/oldest: + get: + summary: get the creation date of the oldest job + operationId: getOldestJob + tags: + - job + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: all_workspaces + description: consider jobs from all workspaces (only valid if request come from the `admins` workspace) + in: query + schema: + type: boolean + responses: + "200": + description: creation date of the oldest job, absent when there is none + content: + application/json: + schema: + type: object + properties: + created_at: + type: string + format: date-time + /w/{workspace}/jobs/completed/count: get: summary: get completed count @@ -17278,6 +17423,27 @@ paths: schema: type: string + /w/{workspace}/jobs/worker_is_draining/{id}: + get: + summary: whether the worker running this job is draining + description: | + True once the worker running the job has received its shutdown signal. The worker + still waits for the job to finish, so a long-running job can poll this to exit early. + False when the job is not currently running on a worker. + operationId: workerIsDraining + tags: + - job + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - $ref: "#/components/parameters/JobId" + responses: + "200": + description: whether the worker running this job is draining + content: + application/json: + schema: + type: boolean + /w/{workspace}/jobs_u/get_logs/{id}: get: summary: get job logs @@ -23458,6 +23624,54 @@ paths: - id - values + /workers/runnable_stats: + get: + summary: get resource usage per runnable and worker group over a time window (require devops role) + operationId: getRunnableStats + tags: + - worker + parameters: + - name: window_secs + in: query + required: false + description: how far back to read, in seconds (defaults to one day, capped at the 30-day retention) + schema: + type: integer + - name: worker_group + in: query + required: false + description: only the jobs run by this worker group + schema: + type: string + - name: workspace + in: query + required: false + description: only the jobs of this workspace + schema: + type: string + - name: order_by + in: query + required: false + description: "what the rows are ranked by, descending (default: total_duration)" + schema: + type: string + enum: [total_duration, max_memory, cpu] + - name: limit + in: query + required: false + description: number of rows to return (default 100, at most 1000) + schema: + type: integer + responses: + "200": + description: resource usage per runnable and worker group, heaviest first + content: + application/json: + schema: + type: array + items: + $ref: "#/components/schemas/RunnableStats" + /workers/queue_metrics_series: get: summary: get the queue metrics of a time window, as a bounded line per tag @@ -25165,9 +25379,12 @@ paths: in: query schema: type: string + - $ref: "#/components/parameters/TablePreviewS3ResourcePath" + - $ref: "#/components/parameters/TablePreviewFilters" responses: "200": - description: Parquet Preview + description: > + Parquet Preview: `columns`, `column_types` (Arrow type names) and `rows`. content: application/json: {} @@ -25192,6 +25409,13 @@ paths: in: query schema: type: string + - $ref: "#/components/parameters/TablePreviewS3ResourcePath" + - name: csv_separator + in: query + schema: + type: string + - $ref: "#/components/parameters/TablePreviewCsvHasHeader" + - $ref: "#/components/parameters/TablePreviewFilters" responses: "200": description: Table count @@ -25240,13 +25464,20 @@ paths: in: query schema: type: string + - $ref: "#/components/parameters/TablePreviewS3ResourcePath" - name: csv_separator in: query schema: type: string + - $ref: "#/components/parameters/TablePreviewCsvHasHeader" + - $ref: "#/components/parameters/TablePreviewFilters" responses: "200": - description: Csv Preview + description: > + Csv Preview: `columns`, `column_types` (Arrow type names), `rows`, and the + `csv_separator` the file was read with, guessed from its first bytes when + none is given. + A `.jsonl` or `.ndjson` file is read as newline-delimited JSON. content: application/json: {} @@ -27606,6 +27837,28 @@ components: required: true schema: type: string + TablePreviewS3ResourcePath: + name: s3_resource_path + description: Read the file from this object storage resource instead of the workspace storage. + in: query + schema: + type: string + TablePreviewCsvHasHeader: + name: csv_has_header + description: Whether the first CSV line names the columns. Defaults to true. + in: query + schema: + type: boolean + TablePreviewFilters: + name: filters + description: > + JSON object, column name to filter. A number column takes an optional comparison + operator and a number (`>=10`), a boolean column a boolean, and any other column a + case-insensitive substring, or `=value` for an exact match. A `search_term` without + a `search_col` matches any column, case-insensitively. + in: query + schema: + type: string HubPublishFolder: name: folder in: query @@ -27788,6 +28041,12 @@ components: in: query schema: type: string + RunRetry: + name: retry + description: JSON-encoded retry policy (same shape as a schedule's retry) re-running the job when it fails. The returned id is the first attempt's; each retry is a job of its own, recorded as a child of the first attempt. Not supported for hub scripts, with invisible_to_owner, or for a script whose preprocessor would run. + in: query + schema: + type: string NewJobId: name: job_id description: @@ -28478,6 +28737,8 @@ components: $ref: "../../openflow.openapi.yaml#/components/schemas/AiTransform" ProviderConfig: $ref: "../../openflow.openapi.yaml#/components/schemas/ProviderConfig" + ProviderTransform: + $ref: "../../openflow.openapi.yaml#/components/schemas/ProviderTransform" MemoryConfig: $ref: "../../openflow.openapi.yaml#/components/schemas/MemoryConfig" FlowModuleValue: @@ -28498,6 +28759,8 @@ components: $ref: "../../openflow.openapi.yaml#/components/schemas/BranchAll" AiAgent: $ref: "../../openflow.openapi.yaml#/components/schemas/AiAgent" + AiDecision: + $ref: "../../openflow.openapi.yaml#/components/schemas/AiDecision" Identity: $ref: "../../openflow.openapi.yaml#/components/schemas/Identity" FlowStatus: @@ -29462,6 +29725,8 @@ components: togetherai, aws_bedrock, customai, + typesafe, + cloudflare, ] GitSyncObjectType: @@ -30150,6 +30415,12 @@ components: items: type: string default: [] + job_token_scopes: + type: array + nullable: true + items: + type: string + description: "Scopes the token of every job of this runnable is restricted to (domain:action[:resource], e.g. oidc:write). Jobs it starts, flow steps and AI agent tools inherit the restriction. Unset, the job token carries the full permissions of the identity the job runs as; an empty list leaves it no API access beyond its own job." inherited_labels: type: array items: @@ -30307,6 +30578,12 @@ components: type: array items: type: string + job_token_scopes: + type: array + nullable: true + items: + type: string + description: "Scopes the token of every job of this runnable is restricted to (domain:action[:resource], e.g. oidc:write). Jobs it starts, flow steps and AI agent tools inherit the restriction. Unset, the job token carries the full permissions of the identity the job runs as; an empty list leaves it no API access beyond its own job. Omitting the field keeps the deployed value; null clears it." skip_draft_deletion: type: boolean description: "When true (set by the CLI / git sync), deploying this script does not delete an existing user draft at the same path." @@ -31642,6 +31919,7 @@ components: - "resource_types.create" - "resource_types.update" - "resource_types.delete" + - "resource_types.sync" - "schedule.create" - "schedule.setenabled" - "schedule.edit" @@ -34473,6 +34751,11 @@ components: type: boolean cluster: $ref: "#/components/schemas/DatatableRoleCluster" + in_use: + type: boolean + description: >- + Whether any data table, in any workspace, lets someone connect as this role. Only set + in a data table's available_roles. DatatableRoleTenants: type: object @@ -34502,6 +34785,7 @@ components: type: boolean default_role: type: string + description: A role id, `admin`, or `$none` when the data table has no default role. roles: type: array items: @@ -35780,6 +36064,45 @@ components: `u/`, `g/`, or an email that exists in this workspace. + RunnableStats: + type: object + description: > + Resource usage of the jobs of one runnable on one worker group. Memory and CPU are + only known for jobs run as a subprocess; memory is the peak of the job's main process. + properties: + workspace_id: + type: string + runnable_path: + type: string + description: path of the script or flow step; `` gathers previews and other runs without a stable path + worker_group: + type: string + job_count: + type: integer + total_duration_ms: + type: integer + max_memory_peak: + type: integer + description: highest peak memory of a job, in kB + sum_memory_peak: + type: integer + description: sum of the peak memory of the jobs that have one, in kB + memory_sample_count: + type: integer + description: number of jobs that have a peak memory reading + total_cpu_ms: + type: integer + required: + - workspace_id + - runnable_path + - worker_group + - job_count + - total_duration_ms + - max_memory_peak + - sum_memory_peak + - memory_sample_count + - total_cpu_ms + WorkerPing: type: object properties: @@ -35828,6 +36151,9 @@ components: type: string native_mode: type: boolean + draining: + type: boolean + description: the worker received its shutdown signal and exits once its current job is done required: - worker - worker_instance @@ -36165,6 +36491,12 @@ components: items: type: string default: [] + job_token_scopes: + type: array + nullable: true + items: + type: string + description: "Scopes the token of every job of this runnable is restricted to (domain:action[:resource], e.g. oidc:write). Jobs it starts, flow steps and AI agent tools inherit the restriction. Unset, the job token carries the full permissions of the identity the job runs as; an empty list leaves it no API access beyond its own job." inherited_labels: type: array items: @@ -36211,6 +36543,12 @@ components: type: array items: type: string + job_token_scopes: + type: array + nullable: true + items: + type: string + description: "Scopes the token of every job of this runnable is restricted to (domain:action[:resource], e.g. oidc:write). Jobs it starts, flow steps and AI agent tools inherit the restriction. Unset, the job token carries the full permissions of the identity the job runs as; an empty list leaves it no API access beyond its own job. Omitting the field keeps the deployed value; null clears it." required: - path # Like OpenFlowWPath but `path` is optional: on update the flow is identified by @@ -36247,6 +36585,12 @@ components: type: array items: type: string + job_token_scopes: + type: array + nullable: true + items: + type: string + description: "Scopes the token of every job of this runnable is restricted to (domain:action[:resource], e.g. oidc:write). Jobs it starts, flow steps and AI agent tools inherit the restriction. Unset, the job token carries the full permissions of the identity the job runs as; an empty list leaves it no API access beyond its own job. Omitting the field keeps the deployed value; null clears it." FlowPreview: type: object @@ -37772,6 +38116,9 @@ components: builder_flows: type: boolean description: Whether operators can compose flows out of existing runnables (consumes a full seat). Omitting the field leaves the stored value unchanged. + builder_apps: + type: boolean + description: Whether operators can compose full-code apps out of existing runnables (consumes a full seat). Omitting the field leaves the stored value unchanged. manage_schedules: type: boolean description: Whether operators can create, edit and delete schedules. Granted unless withdrawn; omitting the field leaves the stored value unchanged. diff --git a/backend/windmill-api/src/agent_runs.rs b/backend/windmill-api/src/agent_runs.rs index d2e76dcca0..76f729e7e0 100644 --- a/backend/windmill-api/src/agent_runs.rs +++ b/backend/windmill-api/src/agent_runs.rs @@ -29,7 +29,7 @@ use crate::db::{ApiAuthed, DB}; use crate::jobs::{handle_chat_conversation_messages, set_flow_memory_id, RunJobQuery}; /// The id the agent's step carries in the run, as the editor's chat names it. A token scoped to -/// `jobs:run:agents:` reads its runs back by it (`require_job_within_run_scope`). +/// `jobs:run:agents:` reads its runs back by it (`require_job_within_read_scope`). const AGENT_NODE_ID: &str = "__wm_agent_root"; /// A turn of the agent's chat when `memory_id` names a conversation, a single run otherwise. @@ -84,6 +84,7 @@ pub(crate) async fn run_agent( )?; let push_authed = authed.clone().into(); + let scope_ceiling = windmill_api_auth::caller_scope_ceiling(&db, &authed).await?; let (uuid, mut tx) = push( &db, PushIsolationLevel::Isolated(user_db.clone(), authed.clone().into()), @@ -118,6 +119,7 @@ pub(crate) async fn run_agent( None, authed.trigger_or_fallback(None), None, + scope_ceiling.as_deref(), ) .await?; diff --git a/backend/windmill-api/src/ai.rs b/backend/windmill-api/src/ai.rs index 855cd29987..cf7447ceba 100644 --- a/backend/windmill-api/src/ai.rs +++ b/backend/windmill-api/src/ai.rs @@ -40,6 +40,7 @@ use windmill_ai::proxy::{ fim::maybe_transform_fim_request, proxy_execution_mode, ProxyBuildArgs, ProxyExecutionMode, ProxyRequest, }; +use windmill_ai::retry::send_with_retries; use windmill_audit::{audit_oss::audit_log, ActionKind}; use windmill_common::db::UserDB; use windmill_common::error::{to_anyhow, Error, Result}; @@ -246,6 +247,12 @@ struct AIStandardResource { /// Custom HTTP headers to include in AI requests #[serde(default)] headers: HashMap, + /// The API token of a `cloudflare` resource, which names it `token` rather than `api_key`. + #[serde(default, deserialize_with = "empty_string_as_none")] + token: Option, + /// The account of a `cloudflare` resource, which its Workers AI URL is built from. + #[serde(default, deserialize_with = "empty_string_as_none")] + account_id: Option, } #[derive(Deserialize, Debug)] @@ -293,10 +300,21 @@ async fn resolve_provider_credentials( // Skip get_base_url for Bedrock - it uses SDK directly, not HTTP let base_url = if matches!(provider, AIProvider::AWSBedrock) { String::new() + } else if *provider == AIProvider::Cloudflare && resource.base_url.is_none() { + let account_id = if let Some(account_id) = resource.account_id { + Some(resolve_var(account_id, db, w_id, user_db.as_ref(), authed).await?) + } else { + None + }; + windmill_ai::ai_providers::cloudflare_workers_ai_base_url(account_id.as_deref())? } else { provider.get_base_url(resource.base_url, db).await? }; - let api_key = if let Some(api_key) = resource.api_key { + let api_key = match provider { + AIProvider::Cloudflare => resource.api_key.or(resource.token), + _ => resource.api_key, + }; + let api_key = if let Some(api_key) = api_key { Some(resolve_var(api_key, db, w_id, user_db.as_ref(), authed).await?) } else { None @@ -486,9 +504,10 @@ async fn assume_bedrock_role( ) -> Result { #[cfg(all(feature = "enterprise", feature = "openidconnect", feature = "private"))] { - use windmill_common::oidc_oss::{generate_id_token, WorkspaceClaim}; + use windmill_common::oidc_oss::{generate_id_token, OidcCaller, WorkspaceClaim}; let id_token = generate_id_token( + OidcCaller::Bedrock, Some(db), WorkspaceClaim { workspace: w_id.to_string() }, windmill_ai::ai_bedrock::AWS_OIDC_AUDIENCE, @@ -1170,15 +1189,11 @@ async fn global_proxy( } }; - let response = request.send().await.map_err(to_anyhow)?; + let response = send_with_retries(request).await; audit_global_ai_request(&db, &authed).await?; - if response.error_for_status_ref().is_err() { - let err_msg = response.text().await.unwrap_or("".to_string()); - return Err(Error::AIError(err_msg)); - } - + let response = response?; let status_code = response.status(); let headers = response.headers().clone(); let stream = response.bytes_stream(); @@ -1238,8 +1253,8 @@ async fn proxy( .get("X-Resource-Value") .map(|v| { let invalid = |e: String| Error::BadRequest(format!("Invalid X-Resource-Value: {e}")); - let decoded = - urlencoding::decode(v.to_str().unwrap_or("")).map_err(|e| invalid(e.to_string()))?; + let decoded = urlencoding::decode(v.to_str().unwrap_or("")) + .map_err(|e| invalid(e.to_string()))?; let value = serde_json::from_str::(&decoded) .map_err(|e| invalid(e.to_string()))?; check_scopes(&authed, || "resources:write".to_string())?; @@ -1558,7 +1573,7 @@ async fn proxy( } }; - let response = request.send().await.map_err(to_anyhow)?; + let response = send_with_retries(request).await; let mut tx = db.begin().await?; @@ -1574,11 +1589,7 @@ async fn proxy( .await?; tx.commit().await?; - if response.error_for_status_ref().is_err() { - let err_msg = response.text().await.unwrap_or("".to_string()); - return Err(Error::AIError(err_msg)); - } - + let response = response?; let status_code = response.status(); let headers = response.headers().clone(); let is_sse = is_sse_response(&headers); diff --git a/backend/windmill-api/src/ai_evals/run.rs b/backend/windmill-api/src/ai_evals/run.rs index 42c2fcb2f2..57e75b3f9f 100644 --- a/backend/windmill-api/src/ai_evals/run.rs +++ b/backend/windmill-api/src/ai_evals/run.rs @@ -760,6 +760,7 @@ async fn push_run_flow( let path = subject.path.clone(); let tx = PushIsolationLevel::Isolated(user_db.clone(), authed.clone().into()); + let scope_ceiling = windmill_api_auth::caller_scope_ceiling(db, authed).await?; let (uuid, tx) = push( db, tx, @@ -790,6 +791,7 @@ async fn push_run_flow( None, authed.trigger_or_fallback(None), None, + scope_ceiling.as_deref(), ) .await?; tx.commit().await?; diff --git a/backend/windmill-api/src/ai_sessions.rs b/backend/windmill-api/src/ai_sessions.rs index 65a8137c4e..9e980a54f4 100644 --- a/backend/windmill-api/src/ai_sessions.rs +++ b/backend/windmill-api/src/ai_sessions.rs @@ -24,7 +24,7 @@ use axum::{ Extension, Json, Router, }; use futures::{StreamExt, TryStreamExt}; -use magic_crypt::{MagicCrypt256, MagicCryptTrait}; +use magic_crypt::MagicCryptTrait; use serde::{Deserialize, Serialize}; use serde_json::value::RawValue; use std::sync::Arc; @@ -35,7 +35,7 @@ use windmill_api_workspaces::ai_session_backups::{ use windmill_api_workspaces::workspaces::sessions_retention_days; use windmill_common::error::{Error, JsonResult, Result}; use windmill_common::utils::calculate_hash; -use windmill_common::variables::{crypt_from_key_with_suffix, get_workspace_key}; +use windmill_common::variables::{crypt_from_key_with_suffix, get_workspace_key, WorkspaceCrypt}; use windmill_object_store::object_store_reexports::{ ObjectStore, ObjectStoreError, Path as ObjectPath, PutPayload, }; @@ -135,7 +135,7 @@ enum Read { /// The user's prefix in the workspace storage, plus what reads and writes it. struct Backend { store: Arc, - mc: MagicCrypt256, + mc: WorkspaceCrypt, prefix: String, /// Name the storage and the generation the objects are under, for the browser's sync /// state: a row recorded against another storage or generation is stale, a removal is diff --git a/backend/windmill-api/src/apps.rs b/backend/windmill-api/src/apps.rs index f01275c953..2c2179d830 100644 --- a/backend/windmill-api/src/apps.rs +++ b/backend/windmill-api/src/apps.rs @@ -56,7 +56,10 @@ use std::str; use windmill_audit::audit_oss::{audit_log, AuditAuthorable}; use windmill_audit::ActionKind; use windmill_common::{ - apps::{AppScriptId, ListAppQuery, APP_WORKSPACED_ROUTE}, + apps::{ + app_value_has_inline_script, app_value_runnable_paths, AppScriptId, ListAppQuery, + APP_WORKSPACED_ROUTE, + }, auth::{APP_EMBED_TOKEN_LABEL_PREFIX, RAW_APP_SDK_TOKEN_LABEL_PREFIX, TOKEN_PREFIX_LEN}, cache::{self, future::FutureCachedExt}, db::{DbWithOptAuthed, UserDB}, @@ -75,7 +78,8 @@ use windmill_common::{ variables::{build_crypt, build_crypt_with_key_suffix, encrypt}, worker::{to_raw_value, CLOUD_HOSTED}, workspaces::{ - check_deploy_rules, check_user_against_rule, ProtectionRuleKind, RuleCheckResult, + check_deploy_rules, check_operator_can_build, check_user_against_rule, + operator_builder_rights, BuilderKind, ProtectionRuleKind, RuleCheckResult, }, HUB_BASE_URL, }; @@ -749,7 +753,7 @@ async fn list_apps( // Append the authed user's `app`/`raw_app` drafts at paths with no deployed app; // see scripts.rs. if lq.include_draft_only.unwrap_or(false) - && !authed.is_operator + && (!authed.is_operator || operator_builder_rights(&db, &w_id).await?.apps) && offset == 0 && lq.path_start.is_none() && lq.path_exact.is_none() @@ -764,6 +768,7 @@ async fn list_apps( path, value as "value!: sqlx::types::Json>", created_at, + email IS NULL as "legacy!", typ::text as "typ!" FROM draft WHERE workspace_id = $1 @@ -813,9 +818,9 @@ async fn list_apps( inherited_labels: None, is_draft: true, draft_path, - // Synthesized rows are the authed user's own draft. + // Owned by nobody when legacy; see scripts.rs. draft_users: Some(sqlx::types::Json(vec![DraftUserRef { - username: Some(authed.username.clone()), + username: (!row.legacy).then(|| authed.username.clone()), }])), }); } @@ -978,7 +983,14 @@ async fn get_raw_app_data( // the opaque, sandboxed wrapper (incl. under a cross-origin-isolated // / COEP `require-corp` embedder). .header("X-Content-Type-Options", "nosniff") - .header("Cross-Origin-Resource-Policy", "cross-origin"); + .header("Cross-Origin-Resource-Policy", "cross-origin") + // The secret names one app version, whose bundle is written once at deploy, + // so the URL's content never changes. `private`: the URL is a capability, so + // shared caches must not keep serving it after the app is deleted. + .header( + http::header::CACHE_CONTROL, + "private, max-age=31536000, immutable", + ); Ok(res.body(body).unwrap()) } else { return Err(Error::NotFound("File not found".to_string())); @@ -1732,11 +1744,14 @@ async fn mint_preview_sdk_token( Path(w_id): Path, Json(req): Json, ) -> Result { - if authed.is_operator { - return Err(Error::NotAuthorized( - "Operators cannot preview raw apps".to_string(), - )); - } + check_operator_can_build( + &db, + &w_id, + authed.is_operator, + BuilderKind::Apps, + "preview raw apps", + ) + .await?; check_scopes(&authed, || format!("apps:write:{}", req.path))?; let (token, _expiration) = mint_raw_app_sdk_token(&db, &w_id, &req.path, &authed, &req.scopes, job_id).await?; @@ -2094,18 +2109,23 @@ async fn get_id_from_secret( prefix: Option<&str>, ) -> Result { let mc = build_crypt(db, w_id).await?; - let decrypted = mc - .decrypt_bytes_to_bytes(&(hex::decode(secret)?)) - .map_err(|e| Error::internal_err(e.to_string()))?; - let mut bytes = str::from_utf8(&decrypted).map_err(to_anyhow)?; - if let Some(prefix) = prefix { - if !bytes.starts_with(prefix) { - return Err(Error::BadRequest("Invalid secret".to_string())); - } - bytes = bytes.strip_prefix(prefix).unwrap_or(""); + id_from_secret(&mc, &secret, prefix) + .ok_or_else(|| Error::BadRequest("Invalid secret".to_string())) +} + +/// The secret comes from the URL of unauthenticated routes, so every way it can be invalid +/// must produce the same response: a caller must never learn why a secret was rejected. +fn id_from_secret( + mc: &magic_crypt::MagicCrypt256, + secret: &str, + prefix: Option<&str>, +) -> Option { + let decrypted = mc.decrypt_bytes_to_bytes(&hex::decode(secret).ok()?).ok()?; + let plaintext = String::from_utf8(decrypted).ok()?; + match prefix { + Some(prefix) => plaintext.strip_prefix(prefix)?.parse().ok(), + None => plaintext.parse().ok(), } - let id: i64 = bytes.parse().map_err(to_anyhow)?; - Ok(id) } async fn get_public_resource( @@ -2254,6 +2274,260 @@ async fn store_raw_app_file<'a>( Ok(()) } +/// A builder app write is held to one set of rules whether it deploys or saves a draft: the +/// editor previews a draft as whoever opens it, and an admin may deploy it as it stands. A deploy +/// fills in what a draft must state, since it stores the policy it checked. +#[derive(Clone, Copy, PartialEq)] +pub(crate) enum BuilderAppWrite { + Deploy, + Draft, +} + +/// The half of the builder-app check that needs no DB (see `docs/operator-builder-rights.md`). +/// Returns every runnable the app can invoke, as `(is_flow, path)`, from both the policy's +/// triggerables and the value's path runnables: they are not the same list. +fn check_operator_composed_app( + write: BuilderAppWrite, + raw_app: bool, + value: Option<&RawValue>, + policy: Option<&mut Policy>, +) -> Result> { + let verb = if write == BuilderAppWrite::Deploy { + "deploy" + } else { + "save" + }; + if !raw_app { + return Err(Error::PermissionDenied( + "Operators with builder rights can only author full-code apps".to_string(), + )); + } + let mut referenced: Vec<(bool, String)> = Vec::new(); + if let Some(value) = value { + // The text stored, NULs stripped: a stripped NUL can rename a key into one this reads. + let value: serde_json::Value = + serde_json::from_str(&strip_json_nul(value.get())).map_err(to_anyhow)?; + if app_value_has_inline_script(&value) { + return Err(Error::PermissionDenied(format!( + "Operators with builder rights cannot {verb} an app carrying inline scripts" + ))); + } + referenced.extend(app_value_runnable_paths(&value)?); + } + let Some(policy) = policy else { + return Err(match write { + BuilderAppWrite::Deploy => Error::BadRequest( + "Operators with builder rights must deploy an app with its policy".to_string(), + ), + BuilderAppWrite::Draft => Error::PermissionDenied( + "Operators with builder rights can only save sandboxed apps".to_string(), + ), + }); + }; + // Every deploy of a draft rebuilds `triggerables_v2` from the runnables, so a draft's copy, + // stale since the last deploy, must not block autosave. A deploy path that skips the rebuild + // must stop exempting it here. + if write == BuilderAppWrite::Draft { + policy.triggerables_v2 = None; + } + referenced.extend(checked_policy_runnable_paths(policy)?); + match (policy.sandbox, write) { + (Some(true), _) => {} + (None, BuilderAppWrite::Deploy) => policy.sandbox = Some(true), + _ => { + return Err(Error::PermissionDenied(format!( + "Operators with builder rights can only {verb} sandboxed apps" + ))) + } + } + + // Viewer mode would make the triggerables collected above non-exhaustive (see the docs), and + // the deploy panel shows it as members-only, so an admin deploying a draft would not see it. + // Pin an omitted mode on deploy: `update_app_internal` resolves it to the deployed app's, so a + // redeploy over an admin's viewer-mode app would otherwise inherit `Viewer`. + match policy.stated_execution_mode() { + Some(ExecutionMode::Viewer) => { + return Err(Error::PermissionDenied(format!( + "Operators with builder rights cannot {verb} an app that runs as its viewer" + ))) + } + None if write == BuilderAppWrite::Deploy => { + policy.set_execution_mode(ExecutionMode::Publisher) + } + _ => {} + } + + referenced.sort(); + referenced.dedup(); + refuse_hub_runnables(&referenced)?; + Ok(referenced) +} + +/// The runnables a builder app's policy authorizes, as `(is_flow, path)`; refuses a policy that +/// pins inline code. +fn checked_policy_runnable_paths(policy: &Policy) -> Result> { + // A `rawscript/` triggerable is the deployed app's authorization to run caller-supplied + // `raw_code` hashing to it: pinning one hands a builder arbitrary code execution through an + // app whose value passed the inline-script check. A composition-only app has none. + fn pins_raw_script(map: &Option>) -> bool { + map.iter() + .flat_map(|m| m.keys()) + .any(|k| k.starts_with("rawscript/") || k.contains(":rawscript/")) + } + if pins_raw_script(&policy.triggerables) || pins_raw_script(&policy.triggerables_v2) { + return Err(Error::PermissionDenied( + "Operators with builder rights cannot save or deploy an app whose policy pins inline code" + .to_string(), + )); + } + // `execute_component` looks up `{component}:{path}` with an unrestricted component, so every + // colon is a possible split (`a:b:script/x` resolves for `component = "a:b"`): check every + // suffix that parses as a runnable. That copies a key once per colon, so bound the key first: + // a 255-char path plus a component name is far below the cap. + const MAX_KEY_LEN: usize = 512; + let keys = || { + policy + .triggerables + .iter() + .flat_map(|t| t.keys()) + .chain(policy.triggerables_v2.iter().flat_map(|t| t.keys())) + }; + if keys().any(|k| k.len() > MAX_KEY_LEN) { + return Err(Error::BadRequest(format!( + "App policy triggerable keys must be at most {MAX_KEY_LEN} bytes" + ))); + } + Ok(keys() + .flat_map(|key| { + std::iter::once(key.as_str()) + .chain(key.match_indices(':').map(|(i, _)| &key[i + 1..])) + .filter_map(|candidate| { + candidate + .strip_prefix("script/") + .map(|p| (false, p.to_string())) + .or_else(|| { + candidate + .strip_prefix("flow/") + .map(|p| (true, p.to_string())) + }) + }) + .collect::>() + }) + .collect()) +} + +fn refuse_hub_runnables(referenced: &[(bool, String)]) -> Result<()> { + for (_, path) in referenced { + if path.starts_with("hub/") { + return Err(Error::PermissionDenied(format!( + "Operators with builder rights cannot reference the hub runnable {path}. Deploy it to the workspace first." + ))); + } + } + Ok(()) +} + +/// Runs on every app write by an operator with builder rights, from inside the create/update +/// internals so both raw-app endpoints are covered by one check. +/// +/// `execute_component` resolves the runnable it picks with the root DB handle, so a path the +/// builder cannot read would still run, and in a `Viewer` app it would run as whoever opened it. +/// RLS on this transaction is the check; refusing `Viewer` above is what makes it exhaustive. +pub(crate) async fn validate_operator_composed_app( + authed: &ApiAuthed, + db: &DB, + user_db: &UserDB, + w_id: &str, + write: BuilderAppWrite, + raw_app: bool, + value: Option<&RawValue>, + policy: Option<&mut Policy>, +) -> Result<()> { + // A deploy resets the run identity to its builder (unless they are in `wm_deployers`, see + // docs/operator-builder-rights.md); a draft keeps whatever it names, which an admin's + // one-click draft deploy preserves without showing it. + if let (BuilderAppWrite::Draft, Some(policy)) = (write, policy.as_deref()) { + let names_another = policy + .on_behalf_of + .as_deref() + .is_some_and(|p| p != username_to_permissioned_as(&authed.username)) + || policy + .on_behalf_of_email + .as_deref() + .is_some_and(|e| e != authed.email); + if names_another { + return Err(Error::PermissionDenied( + "Operators with builder rights cannot save an app that runs as someone else" + .to_string(), + )); + } + } + let referenced = check_operator_composed_app(write, raw_app, value, policy)?; + require_runnables_readable(authed, db, user_db, w_id, referenced).await +} + +async fn require_runnables_readable( + authed: &ApiAuthed, + db: &DB, + user_db: &UserDB, + w_id: &str, + referenced: Vec<(bool, String)>, +) -> Result<()> { + if referenced.is_empty() { + return Ok(()); + } + // Each script version carries its own grants, and a path reused after its versions were + // archived starts with none. Check the version a run would pick, found without RLS, not + // whichever version the caller happens to see. Resolved before the transaction opens so no + // request holds two pool connections at once. + let mut hashes = Vec::with_capacity(referenced.len()); + for (is_flow, path) in &referenced { + hashes.push(if *is_flow { + None + } else { + windmill_common::get_latest_script_hash(db, path, w_id).await? + }); + } + let mut tx = user_db.clone().begin(authed).await?; + for ((is_flow, path), hash) in referenced.into_iter().zip(hashes) { + let readable = if is_flow { + sqlx::query_scalar!( + "SELECT EXISTS(SELECT 1 FROM flow WHERE workspace_id = $1 AND path = $2)", + w_id, + path, + ) + .fetch_one(&mut *tx) + .await? + .unwrap_or(false) + } else { + match hash { + Some(hash) => sqlx::query_scalar!( + "SELECT EXISTS(SELECT 1 FROM script WHERE workspace_id = $1 AND hash = $2)", + w_id, + hash, + ) + .fetch_one(&mut *tx) + .await? + .unwrap_or(false), + None => false, + } + }; + // One message for every cause: telling "no runnable version" apart from "not readable" + // would reveal that a path the caller cannot read exists. + if !readable { + return Err(Error::PermissionDenied(if is_flow { + format!("Flow {path} does not exist or is not readable by you") + } else { + format!( + "Script {path} does not exist, is not readable by you, or has not finished deploying" + ) + })); + } + } + tx.commit().await?; + Ok(()) +} + macro_rules! process_app_multipart { ($authed:expr, $user_db:expr, $db:expr, $w_id:expr, $path:expr, $multipart:expr, $internal_fn:expr) => { async { @@ -2330,11 +2604,14 @@ async fn create_app_raw<'a>( Path(w_id): Path, multipart: Multipart, ) -> Result<(StatusCode, String)> { - if authed.is_operator { - return Err(Error::NotAuthorized( - "Operators cannot create apps for security reasons".to_string(), - )); - } + check_operator_can_build( + &db, + &w_id, + authed.is_operator, + BuilderKind::Apps, + "create apps", + ) + .await?; if let RuleCheckResult::Blocked(msg) = check_deploy_rules( &w_id, @@ -2511,6 +2788,19 @@ async fn create_app_internal<'a>( if raw_app { validate_raw_app_path_keys(&app.value.0)?; } + if authed.is_operator { + validate_operator_composed_app( + &authed, + &db, + &user_db, + w_id, + BuilderAppWrite::Deploy, + raw_app, + Some(&app.value.0), + Some(&mut app.policy), + ) + .await?; + } if *CLOUD_HOSTED { let nb_apps = sqlx::query_scalar!("SELECT COUNT(*) FROM app WHERE workspace_id = $1", &w_id) @@ -2755,6 +3045,7 @@ async fn create_app_internal<'a>( None, None, None, + None, ) .await?; tracing::info!("Pushed app dependency job {}", dependency_job_uuid); @@ -2816,13 +3107,23 @@ async fn delete_app( Extension(webhook): Extension, Path((w_id, path)): Path<(String, StripPath)>, ) -> Result { - if authed.is_operator { - return Err(Error::NotAuthorized( - "Operators cannot delete apps for security reasons".to_string(), - )); - } + check_operator_can_build( + &db, + &w_id, + authed.is_operator, + BuilderKind::Apps, + "delete apps", + ) + .await?; let path = path.to_path(); check_scopes(&authed, || format!("apps:write:{}", path))?; + // One endpoint deletes both kinds, and builder rights only cover full-code apps. + if authed.is_operator && deployed_app_kind(&user_db, &authed, &w_id, path).await? != Some(true) + { + return Err(Error::PermissionDenied( + "Operators with builder rights can only delete full-code apps".to_string(), + )); + } if path == "g/all/setup_app" && w_id == "admins" { return Err(Error::BadRequest( @@ -3010,13 +3311,20 @@ async fn update_app( Path((w_id, path)): Path<(String, StripPath)>, Json(ns): Json, ) -> JsonResult { - if authed.is_operator { - return Err(Error::NotAuthorized( - "Operators cannot update apps for security reasons".to_string(), - )); - } - // create_app_internal(authed, user_db, db, &w_id, &mut app).await?; let path = path.to_path(); + // The deploy panel saves a deployed full-code app's access and frontend scopes here. A + // builder may change those settings; the app itself only through `update_app_raw`. + if authed.is_operator { + check_operator_can_build(&db, &w_id, true, BuilderKind::Apps, "update apps").await?; + if ns.value.is_some() + || deployed_app_kind(&user_db, &authed, &w_id, path).await? != Some(true) + { + return Err(Error::PermissionDenied( + "Operators with builder rights can only change the settings of full-code apps" + .to_string(), + )); + } + } check_scopes(&authed, || format!("apps:write:{}", path))?; if let RuleCheckResult::Blocked(msg) = check_deploy_rules( @@ -3033,8 +3341,10 @@ async fn update_app( let opath = path.to_string(); let db2 = db.clone(); + // Settings-only for an operator, of an app checked above to be full-code. + let raw_app = authed.is_operator; let (new_tx, npath, v_id) = - update_app_internal(authed, db, user_db, &w_id, path, false, ns, None).await?; + update_app_internal(authed, db, user_db, &w_id, path, raw_app, ns, None).await?; new_tx.commit().await?; tally_app_rename(&db2, &w_id, &opath, &npath, v_id).await; @@ -3063,6 +3373,10 @@ async fn update_app_raw_source( Path((w_id, path)): Path<(String, StripPath)>, Json(ns): Json, ) -> JsonResult { + // Stays closed to operators even with builder rights, unlike `create_app_raw`/`update_app_raw` + // next to it: this path compiles caller-supplied sources with a bundler job on a worker, which + // is arbitrary code execution. Builders lose nothing: the browser and the CLI both bundle + // locally and deploy through the multipart endpoints. if authed.is_operator { return Err(Error::NotAuthorized( "Operators cannot update apps for security reasons".to_string(), @@ -3223,6 +3537,7 @@ async fn create_app_raw_source( Path(w_id): Path, Json(mut app): Json, ) -> Result<(StatusCode, String)> { + // Closed to builders too, for the reason on `update_app_raw_source`. if authed.is_operator { return Err(Error::NotAuthorized( "Operators cannot create apps for security reasons".to_string(), @@ -3388,11 +3703,14 @@ async fn update_app_raw<'a>( Path((w_id, path)): Path<(String, StripPath)>, multipart: Multipart, ) -> JsonResult { - if authed.is_operator { - return Err(Error::NotAuthorized( - "Operators cannot update apps for security reasons".to_string(), - )); - } + check_operator_can_build( + &db, + &w_id, + authed.is_operator, + BuilderKind::Apps, + "update apps", + ) + .await?; if let RuleCheckResult::Blocked(msg) = check_deploy_rules( &w_id, @@ -3408,6 +3726,14 @@ async fn update_app_raw<'a>( let path = path.to_path(); check_scopes(&authed, || format!("apps:write:{}", path))?; + // `update_app_internal` only compares kinds when a new value is deployed, so a + // policy-only update would otherwise let a builder rewrite a low-code app's policy. + if authed.is_operator && deployed_app_kind(&user_db, &authed, &w_id, path).await? != Some(true) + { + return Err(Error::PermissionDenied( + "Operators with builder rights can only update full-code apps".to_string(), + )); + } let opath = path.to_string(); let db2 = db.clone(); let (npath, v_id) = process_app_multipart!( @@ -3475,6 +3801,23 @@ async fn update_app_internal<'a>( } } + if authed.is_operator { + // Restoring a full-code version always sends allow_kind_change. A builder converts + // nothing, so drop it and let the locked kind check below refuse an actual conversion. + ns.allow_kind_change = None; + validate_operator_composed_app( + &authed, + &db, + &user_db, + w_id, + BuilderAppWrite::Deploy, + raw_app, + ns.value.as_ref().map(|v| v.0.as_ref()), + ns.policy.as_mut(), + ) + .await?; + } + // Resolved on the (non-RLS) pool before the RLS transaction opens, for the reason // `create_app` states, with the same known, accepted rename race (see // `resolve_on_behalf_of`). Submitting a policy is how a deployer claims the app's execution @@ -3896,6 +4239,7 @@ async fn update_app_internal<'a>( None, None, None, + None, ) .await?; tracing::info!("Pushed app dependency job {}", dependency_job_uuid); @@ -4049,10 +4393,55 @@ lazy_static! { /// change or deletion, which don't bump the version — see /// `invalidate_app_policy_cache` and `process_notify_event`); /// - the entry is within its TTL (backstop for a missed event or a restart). - static ref APP_POLICY_CACHE: cache::Cache<(String, String), (Option, Arc, std::time::Instant)> = + static ref APP_POLICY_CACHE: cache::Cache<(String, String), (Option, Arc, std::time::Instant)> = cache::Cache::new(1000); } +/// What `execute_component` runs a deployed app against. +struct DeployedPolicy { + policy: Policy, + /// Runnable id -> argument -> context property (`username`, `email`, `groups`, + /// `workspace`, `author`), for every `ctx` field the app's runnables declare. Read + /// from the deployed app rather than taken from the request, which only says what + /// the browser chose to send: a runnable filters rows on these. A deploy that + /// changes them under an unchanged policy evicts nothing, so a caller still naming + /// the replaced version keeps the previous declarations until the entry's TTL. + ctx_inputs: HashMap>, +} + +async fn deployed_ctx_inputs( + db: &DB, + w_id: &str, + path: &str, +) -> Result>> { + let runnables = sqlx::query_scalar::<_, Option>>( + "SELECT app_version.value -> 'runnables' FROM app + JOIN app_version ON app_version.id = app.versions[array_upper(app.versions, 1)] + WHERE app.path = $1 AND app.workspace_id = $2", + ) + .bind(path) + .bind(w_id) + .fetch_optional(db) + .await? + .flatten(); + let Some(serde_json::Value::Object(runnables)) = runnables.map(|r| r.0) else { + return Ok(HashMap::new()); + }; + Ok(runnables + .into_iter() + .filter_map(|(id, runnable)| { + let ctx_inputs: HashMap = runnable + .get("fields")? + .as_object()? + .iter() + .filter(|(_, f)| f.get("type").and_then(|t| t.as_str()) == Some("ctx")) + .filter_map(|(arg, f)| Some((arg.clone(), f.get("ctx")?.as_str()?.to_string()))) + .collect(); + (!ctx_inputs.is_empty()).then_some((id, ctx_inputs)) + }) + .collect()) +} + /// Backstop time-to-live for an [`APP_POLICY_CACHE`] entry — the maximum a policy /// change can go unreflected if its invalidation event is never consumed. const APP_POLICY_CACHE_TTL: std::time::Duration = std::time::Duration::from_secs(300); @@ -4105,7 +4494,7 @@ async fn execute_component( _ => {} }; - let arc_policy: Arc; + let arc_policy: Arc; let policy_triggerables_default = Default::default(); // Preview mode means the request was issued from the editor; the editing // user is already trusted by the policy check, so client-supplied `tag` @@ -4130,10 +4519,26 @@ async fn execute_component( let authed = opt_authed.as_ref().ok_or_else(|| { Error::NotAuthorized("App component preview requires authentication".to_string()) })?; + // A builder testing the app it is composing only ever previews a deployed runnable + // (`path`), confined below to what it may read. Inline `raw_code`, which an `id` also + // requires, is authoring code, so it stays closed to every operator. So is a hub + // path, which `require_path_read_access_for_preview` admits for everyone and + // `get_payload_tag_from_prefixed_path` then downloads and enqueues: it is exactly the + // unreviewed code the composition check refuses in a flow. + let previews_hub = payload.path.as_deref().is_some_and(|p| { + p.strip_prefix("script/") + .or_else(|| p.strip_prefix("flow/")) + .unwrap_or(p) + .starts_with("hub/") + }); if authed.is_operator { - return Err(Error::NotAuthorized( - "Operators cannot run preview jobs for security reasons".to_string(), - )); + let msg = "Operators cannot run preview jobs for security reasons".to_string(); + if !operator_builder_rights(&db, &w_id).await?.apps { + return Err(Error::NotAuthorized(msg)); + } + if payload.raw_code.is_some() || previews_hub { + return Err(Error::PermissionDenied(msg)); + } } check_scopes(authed, || format!("jobs:run"))?; if let Some(p) = payload.path.as_deref() { @@ -4172,7 +4577,7 @@ async fn execute_component( // - The policy and triggerables are fetched from the database. // - In case of inline script, if an entry exists in the `app_script` table, push // an `AppScript` job payload, as in (.1) otherwise. - let (policy, policy_triggerables) = match payload { + let (policy, policy_triggerables, ctx_inputs) = match payload { // 1. "preview" mode. ExecuteApp { force_viewer_static_fields: Some(static_inputs), @@ -4191,6 +4596,7 @@ async fn execute_component( sensitive_inputs: force_viewer_sensitive_inputs.unwrap_or_default(), tag: None, }, + None, ), // 2. "run" mode. _ => { @@ -4220,9 +4626,12 @@ async fn execute_component( .map(|(_, p, _)| p); let policy = if let Some(p) = fresh { arc_policy = p; - &*arc_policy + &arc_policy.policy } else { - arc_policy = Arc::new(policy_fut.await?); + arc_policy = Arc::new(DeployedPolicy { + policy: policy_fut.await?, + ctx_inputs: deployed_ctx_inputs(&db, &w_id, path).await?, + }); APP_POLICY_CACHE.insert( cache_key, ( @@ -4231,7 +4640,7 @@ async fn execute_component( std::time::Instant::now(), ), ); - &*arc_policy + &arc_policy.policy }; // Caller-supplied inline code (`raw_code`), with or without an @@ -4293,7 +4702,11 @@ async fn execute_component( }) .ok_or_else(|| Error::BadRequest(format!("Path {path} forbidden by policy")))?; - (policy, policy_triggerables) + ( + policy, + policy_triggerables, + arc_policy.ctx_inputs.get(&payload.component), + ) } }; @@ -4371,6 +4784,7 @@ async fn execute_component( let (mut args, job_id) = build_args( policy, policy_triggerables, + ctx_inputs, payload.args, opt_authed.as_ref(), &user_db, @@ -4490,6 +4904,10 @@ async fn execute_component( let app_trigger = (!is_preview).then(|| TriggerMetadata::new(Some(path.to_string()), JobTriggerKind::App)); + let scope_ceiling = match opt_authed.as_ref() { + Some(authed) => windmill_api_auth::caller_scope_ceiling(&db, authed).await?, + None => None, + }; let (uuid, mut tx) = push( &db, tx, @@ -4525,6 +4943,7 @@ async fn execute_component( end_user_email, app_trigger, None, + scope_ceiling.as_deref(), ) .await?; @@ -5483,6 +5902,10 @@ impl AppLoadCountQuery { search_col: self.search_col, search_term: self.search_term, storage: self.storage, + s3_resource_path: None, + csv_separator: None, + csv_has_header: None, + filters: None, }, ) } @@ -5516,6 +5939,9 @@ impl AppLoadPreviewQuery { search_term: self.search_term, storage: self.storage, csv_separator: self.csv_separator, + s3_resource_path: None, + csv_has_header: None, + filters: None, }, ) } @@ -5603,7 +6029,9 @@ async fn app_load_table_count( let job_authed = app_s3_on_behalf_and_provenance(&db, &path, &w_id, &opt_authed, &file_query).await?; let resp = - crate::job_helpers_oss::load_table_count_internal(job_authed, &db, &w_id, file_key, inner) + crate::job_helpers_oss::load_table_count_internal( + job_authed, &db, None, &w_id, file_key, inner, + ) .await?; Ok(Json(resp).into_response()) } @@ -5622,7 +6050,7 @@ async fn app_load_parquet_preview( let job_authed = app_s3_on_behalf_and_provenance(&db, &path, &w_id, &opt_authed, &file_query).await?; let resp = crate::job_helpers_oss::load_preview_internal( - job_authed, &db, &w_id, file_key, inner, true, + job_authed, &db, None, &w_id, file_key, inner, true, ) .await?; Ok(Json(resp).into_response()) @@ -5642,7 +6070,7 @@ async fn app_load_csv_preview( let job_authed = app_s3_on_behalf_and_provenance(&db, &path, &w_id, &opt_authed, &file_query).await?; let resp = crate::job_helpers_oss::load_preview_internal( - job_authed, &db, &w_id, file_key, inner, false, + job_authed, &db, None, &w_id, file_key, inner, false, ) .await?; Ok(Json(resp).into_response()) @@ -5787,6 +6215,54 @@ fn policy_granted_viewer(authed: &ApiAuthed) -> ApiAuthed { ApiAuthed { scopes: None, ..authed.clone() } } +/// The value of an app context property for this run. Everything about the viewer comes +/// from `authed`, the session the request authenticated with; an anonymous run has none +/// and gets `null`. +async fn ctx_value( + prop: &str, + policy: &Policy, + authed: Option<&ApiAuthed>, + w_id: &str, + db: &DB, +) -> Result> { + let value = match prop { + "username" => authed.map(|a| serde_json::to_value(a.display_username())), + "email" => authed.map(|a| serde_json::to_value(&a.email)), + "workspace" => Some(serde_json::to_value(w_id)), + "groups" => authed.map(|a| serde_json::to_value(&a.groups)), + // Same rule as `get_on_behalf_of`: the stored address, derived only when absent. + "author" => { + let author = match ( + policy.on_behalf_of_email.as_deref(), + policy.on_behalf_of.as_deref(), + ) { + (Some(email), _) => Some(email.to_string()), + (None, Some(permissioned_as)) => Some( + windmill_common::users::get_email_from_permissioned_as( + permissioned_as, + w_id, + db, + ) + .await?, + ), + (None, None) => None, + }; + Some(serde_json::to_value(&author)) + } + _ => { + return Err(Error::BadRequest(format!( + "context variable {} not allowed", + prop + ))) + } + }; + Ok(to_raw_value( + &value.unwrap_or(Ok(serde_json::Value::Null)).map_err(|e| { + Error::internal_err(format!("failed to serialize ctx variable {}: {}", prop, e)) + })?, + )) +} + async fn build_args( policy: &Policy, PolicyTriggerableInputs { @@ -5796,6 +6272,7 @@ async fn build_args( sensitive_inputs, .. }: &PolicyTriggerableInputs, + ctx_inputs: Option<&HashMap>, mut args: HashMap>, authed: Option<&ApiAuthed>, user_db: &UserDB, @@ -5805,6 +6282,16 @@ async fn build_args( let mut job_id: Option = None; let mut safe_args = HashMap::>::new(); + // Before anything reads `args`: a context input the runnable declares is never taken + // from the request, or a viewer could name someone else as themselves. + for (k, prop) in ctx_inputs.into_iter().flatten() { + args.remove(k); + safe_args.insert( + k.to_string(), + ctx_value(prop, policy, authed, w_id, db).await?, + ); + } + // tracing::error!("{:?}", allow_user_resources); for k in allow_user_resources.iter() { if let Some(arg_val) = args.get(k) { @@ -5896,47 +6383,9 @@ async fn build_args( if arg_str.starts_with("\"$ctx:") { let prop = arg_str.trim_start_matches("\"$ctx:").trim_end_matches("\""); - let value = match prop { - "username" => authed - .as_ref() - .map(|a| serde_json::to_value(a.display_username())), - "email" => authed.as_ref().map(|a| serde_json::to_value(&a.email)), - "workspace" => Some(serde_json::to_value(&w_id)), - "groups" => authed.as_ref().map(|a| serde_json::to_value(&a.groups)), - // Same rule as `get_on_behalf_of`: the stored address, derived only when absent. - "author" => { - let author = match ( - policy.on_behalf_of_email.as_deref(), - policy.on_behalf_of.as_deref(), - ) { - (Some(email), _) => Some(email.to_string()), - (None, Some(permissioned_as)) => Some( - windmill_common::users::get_email_from_permissioned_as( - permissioned_as, - w_id, - db, - ) - .await?, - ), - (None, None) => None, - }; - Some(serde_json::to_value(&author)) - } - _ => { - return Err(Error::BadRequest(format!( - "context variable {} not allowed", - prop - ))) - } - }; safe_args.insert( k.to_string(), - to_raw_value(&value.unwrap_or(Ok(serde_json::Value::Null)).map_err(|e| { - Error::internal_err(format!( - "failed to serialize ctx variable for {}: {}", - k, e - )) - })?), + ctx_value(prop, policy, authed, w_id, db).await?, ); } else if !arg_str.contains("\"$var:") && !arg_str.contains("\"$res:") { safe_args.insert(k.to_string(), v); @@ -6331,6 +6780,127 @@ mod embed_token_tests { } } +#[cfg(test)] +mod operator_app_tests { + use super::{check_operator_composed_app, BuilderAppWrite, Policy}; + use windmill_common::error::Result; + use windmill_common::worker::to_raw_value; + + fn builder_policy(triggerables_v2: serde_json::Value) -> Policy { + serde_json::from_value(serde_json::json!({ + "execution_mode": "publisher", + "triggerables_v2": triggerables_v2, + })) + .unwrap() + } + + fn path_runnable(path: &str, run_type: &str) -> serde_json::Value { + serde_json::json!({"files": {}, "runnables": { + "a": {"name": "a", "type": "runnableByPath", "path": path, "runType": run_type} + }}) + } + + fn composed_app(value: serde_json::Value, policy: &mut Policy) -> Result> { + let value = to_raw_value(&value); + check_operator_composed_app(BuilderAppWrite::Deploy, true, Some(&value), Some(policy)) + } + + #[test] + fn operator_app_check_forces_the_sandbox_and_refuses_code() { + let clean = serde_json::json!({"files": {}, "runnables": { + "a": {"name": "a", "type": "runnableByPath", "path": "f/x/s", "runType": "script"} + }}); + + // A composition-only app goes through, sandboxed whether or not it asked to be. + let mut policy = builder_policy( + serde_json::json!({"a:script/f/x/s": {"static_inputs": {}, "one_of_inputs": {}}}), + ); + let referenced = composed_app(clean.clone(), &mut policy).unwrap(); + assert_eq!(policy.sandbox, Some(true)); + // The runnables the policy authorizes are handed back for the caller to authorize. + assert_eq!(referenced, vec![(false, "f/x/s".to_string())]); + + // A hub reference is unreviewed code, refused like it is in a flow. + let mut policy = builder_policy( + serde_json::json!({"a:script/hub/1/x": {"static_inputs": {}, "one_of_inputs": {}}}), + ); + assert!(composed_app(clean.clone(), &mut policy).is_err()); + + let mut policy = builder_policy(serde_json::json!({})); + policy.sandbox = Some(false); + assert!(composed_app(clean.clone(), &mut policy).is_err()); + + // An inline script is refused even with no `language`, which the locking traversal skips. + let mut policy = builder_policy(serde_json::json!({})); + assert!(composed_app( + serde_json::json!({"runnables": {"a": {"inlineScript": {"content": "x"}}}}), + &mut policy + ) + .is_err()); + + // A `rawscript/` triggerable is what authorizes caller-supplied `raw_code` on the + // deployed app, so pinning one would be arbitrary code execution behind a clean value. + for key in ["rawscript/abc", "a:rawscript/abc"] { + let mut policy = builder_policy( + serde_json::json!({ key: {"static_inputs": {}, "one_of_inputs": {}} }), + ); + assert!( + composed_app(clean.clone(), &mut policy).is_err(), + "{key} must be refused" + ); + } + + // What the deployed bundle actually asks to run comes from the value's `runnableByPath` + // entries, which are a separate surface from the policy: both are reported. + let mut policy = builder_policy(serde_json::json!({})); + assert_eq!( + composed_app(path_runnable("f/x/s", "script"), &mut policy).unwrap(), + vec![(false, "f/x/s".to_string())] + ); + let mut policy = builder_policy(serde_json::json!({})); + assert_eq!( + composed_app(path_runnable("f/x/f", "flow"), &mut policy).unwrap(), + vec![(true, "f/x/f".to_string())] + ); + let mut policy = builder_policy(serde_json::json!({})); + assert!(composed_app(path_runnable("hub/1/x", "hubscript"), &mut policy).is_err()); + + // `execute_component` looks up `:` with an unrestricted component, so a + // second colon must not hide the path from validation: `x:y:script/hub/…` resolves at run + // time for `component = "x:y"`. + let mut policy = builder_policy( + serde_json::json!({"x:y:script/hub/123/foo": {"static_inputs": {}, "one_of_inputs": {}}}), + ); + assert!(composed_app(clean.clone(), &mut policy).is_err()); + let mut policy = builder_policy( + serde_json::json!({"x:y:script/f/x/s": {"static_inputs": {}, "one_of_inputs": {}}}), + ); + assert_eq!( + composed_app(clean.clone(), &mut policy).unwrap(), + vec![(false, "f/x/s".to_string())] + ); + + // `Viewer` mode makes `execute_component` accept any script/flow path, triggerables or + // not, and run it as the viewer, so the checks above would stop binding. + let mut policy: Policy = serde_json::from_value(serde_json::json!({ + "execution_mode": "viewer", "triggerables_v2": {} + })) + .unwrap(); + assert!(composed_app(clean.clone(), &mut policy).is_err()); + + // Low-code apps stay closed. + let value = to_raw_value(&clean); + let mut policy = builder_policy(serde_json::json!({})); + assert!(check_operator_composed_app( + BuilderAppWrite::Deploy, + false, + Some(&value), + Some(&mut policy) + ) + .is_err()); + } +} + #[cfg(test)] mod policy_tests { /// `execution_mode` is optional in the OpenAPI schema the API clients and the @@ -6353,6 +6923,39 @@ mod policy_tests { } } +#[cfg(test)] +mod app_secret_tests { + use super::{id_from_secret, BUNDLE_SECRET_PREFIX}; + use magic_crypt::MagicCryptTrait; + + #[test] + fn every_invalid_secret_is_rejected_alike() { + let mc = magic_crypt::new_magic_crypt!("workspace-key", 256); + let other = magic_crypt::new_magic_crypt!("another-key", 256); + let valid = hex::encode(mc.encrypt_str_to_bytes("42")); + assert_eq!(id_from_secret(&mc, &valid, None), Some(42)); + let bundle = hex::encode(mc.encrypt_str_to_bytes(format!("{BUNDLE_SECRET_PREFIX}7"))); + assert_eq!( + id_from_secret(&mc, &bundle, Some(BUNDLE_SECRET_PREFIX)), + Some(7) + ); + + let mut tampered = mc.encrypt_str_to_bytes("42"); + *tampered.last_mut().unwrap() ^= 1; + for bad in [ + "not hex".to_string(), + hex::encode(&tampered), + hex::encode(other.encrypt_str_to_bytes("42")), + hex::encode(mc.encrypt_bytes_to_bytes(&[0xff, 0xfe])), + hex::encode(mc.encrypt_str_to_bytes("not a number")), + valid.clone(), + ] { + let prefix = (bad == valid).then_some(BUNDLE_SECRET_PREFIX); + assert_eq!(id_from_secret(&mc, &bad, prefix), None, "{bad}"); + } + } +} + #[cfg(test)] mod raw_app_file_key_tests { use super::validate_raw_app_path_keys; diff --git a/backend/windmill-api/src/apps_raw_bundle.rs b/backend/windmill-api/src/apps_raw_bundle.rs index f50f94f8eb..61dc4b861d 100644 --- a/backend/windmill-api/src/apps_raw_bundle.rs +++ b/backend/windmill-api/src/apps_raw_bundle.rs @@ -148,6 +148,7 @@ pub(crate) async fn bundle_raw_app_sources( args.insert("runnables".to_string(), runnables.to_owned()); let tx = PushIsolationLevel::Isolated(user_db.clone(), authed.clone().into()); + let scope_ceiling = windmill_api_auth::caller_scope_ceiling(db, authed).await?; let (uuid, tx) = push( db, tx, @@ -191,6 +192,7 @@ pub(crate) async fn bundle_raw_app_sources( None, authed.trigger_or_fallback(None), None, + scope_ceiling.as_deref(), ) .await?; tx.commit().await?; diff --git a/backend/windmill-api/src/apps_raw_bundler.ts b/backend/windmill-api/src/apps_raw_bundler.ts index 6663a0f83e..965a9289b5 100644 --- a/backend/windmill-api/src/apps_raw_bundler.ts +++ b/backend/windmill-api/src/apps_raw_bundler.ts @@ -6,7 +6,7 @@ * The build itself is `wmill app bundle`, the same one `wmill app push` runs, so * an app deployed through the API is compiled exactly as the CLI and the editor * compile it — entry point, virtual `wmill` module, `/ui/` shared-UI resolution - * and the Svelte/Vue plugins included. Reimplementing any of that here would be + * and the Svelte plugin included. Reimplementing any of that here would be * a third bundler to keep in step with the other two. */ declare const __wmillAppPolicy: { diff --git a/backend/windmill-api/src/concurrency_groups.rs b/backend/windmill-api/src/concurrency_groups.rs index c94608ac7e..aab73c81d9 100644 --- a/backend/windmill-api/src/concurrency_groups.rs +++ b/backend/windmill-api/src/concurrency_groups.rs @@ -56,6 +56,7 @@ async fn get_concurrency_key( .get_authed(Some(job.workspace_id.clone()), &token) .await .ok_or_else(not_found)?; + windmill_api_auth::check_job_token_scope(&authed_in_workspace, || "jobs:read".to_string())?; require_job_read_access( &db, diff --git a/backend/windmill-api/src/drafts.rs b/backend/windmill-api/src/drafts.rs index 7b2268e38f..e6b6db6919 100644 --- a/backend/windmill-api/src/drafts.rs +++ b/backend/windmill-api/src/drafts.rs @@ -15,15 +15,16 @@ use axum::{ }; use serde::{Deserialize, Serialize}; use windmill_api_flows::flows::validate_operator_flow; +use windmill_audit::{audit_oss::audit_log, ActionKind}; use windmill_common::{ db::UserDB, error::{Error, Result}, flows::FlowValue, user_drafts::{DraftUserRef, UserDraftItemKind, ENCRYPTED_DRAFT_PREFIX}, users::resolve_username_to_email, - utils::{check_proper_path, strip_json_nul}, + utils::{check_proper_path, require_admin, strip_json_nul}, variables::{build_crypt, encrypt}, - workspaces::operator_can_build_flows, + workspaces::{operator_builder_rights, BuilderKind}, }; pub fn workspaced_service() -> Router { @@ -32,6 +33,10 @@ pub fn workspaced_service() -> Router { .route("/get/{kind}/{*path}", get(get_draft_for_user)) .route("/get_own/{kind}/{*path}", get(get_own_draft)) .route("/update/{kind}/{*path}", post(update_draft)) + .route( + "/delete_for_user/{kind}/{*path}", + post(delete_draft_for_user), + ) .route("/move/{kind}/{*path}", post(move_draft)) .route("/migrate_legacy/{kind}/{*path}", post(migrate_legacy_draft)) } @@ -107,7 +112,7 @@ async fn list_drafts( // Without builder rights an operator has no drafts of their own (they can't write any, see // `require_can_write_path`), so this list is always empty for them. They can still READ some // collaborators' drafts via `/drafts/get`. - if authed.is_operator && !operator_can_build_flows(&db, &w_id).await? { + if authed.is_operator && !operator_builder_rights(&db, &w_id).await?.any() { return Ok(Json(vec![])); } let all_users = query.all_users.unwrap_or(false); @@ -496,6 +501,31 @@ async fn update_draft( } } + // A raw-app draft carries the app value and its policy side by side. + if authed.is_operator && kind == UserDraftItemKind::RawApp { + if let Some(value) = &req.value { + let draft: serde_json::Value = serde_json::from_str(&strip_json_nul(value.0.get())) + .map_err(|e| Error::BadRequest(format!("Invalid app draft: {e}")))?; + let mut policy = draft + .get("policy") + .filter(|p| !p.is_null()) + .map(|p| serde_json::from_value::(p.clone())) + .transpose() + .map_err(|e| Error::BadRequest(format!("Invalid app draft policy: {e}")))?; + crate::apps::validate_operator_composed_app( + &authed, + &db, + &user_db, + &w_id, + crate::apps::BuilderAppWrite::Draft, + true, + Some(&value.0), + policy.as_mut(), + ) + .await?; + } + } + let applied = if let Some(value) = &req.value { // Secret variable values must never sit in `draft.value` in plaintext // (see `encrypt_secret_variable_value`). @@ -884,9 +914,9 @@ async fn move_draft( {attempted}. Reopen it, re-save to rewrite it cleanly, then retry." ) } else if row.legacy { - // The home list synthesizes a draft-only row for the legacy draft with the - // caller's own name on it, so this is reachable from the row menu. Only an - // admin can claim or discard that row, and only from the drafts page. + // The home list surfaces the legacy draft as a draft-only row owned by nobody, + // so this is reachable from the row menu. It can be discarded (a write-gated + // legacy delete) or claimed by an admin, but never moved. format!( "'{path}' is a legacy workspace draft with no owner, so it cannot be moved. \ A workspace admin can claim or discard it on the Review & deploy drafts page." @@ -1000,6 +1030,85 @@ async fn migrate_legacy_draft( } } +#[derive(Deserialize, Debug)] +pub struct DeleteDraftForUserQuery { + /// Workspace username of the draft owner. Absent: every draft at the path, + /// the legacy ownerless one and the caller's own included. + pub username: Option, +} + +/// Delete ANOTHER user's draft at a path, or every draft there, for workspace +/// admins (and superadmins, which carry `is_admin` in a workspace). Covers the +/// kinds whose authors are visible to others (`shares_drafts_across_users`); the +/// owner is named by workspace username, as on `/drafts/get`. +async fn delete_draft_for_user( + authed: ApiAuthed, + Extension(db): Extension, + Path((w_id, kind, path)): Path<(String, UserDraftItemKind, windmill_common::utils::StripPath)>, + Query(query): Query, +) -> Result { + require_admin(authed.is_admin, &authed.username)?; + if !kind.shares_drafts_across_users() { + return Err(Error::NotFound( + "drafts for this item kind are private to their owner".to_string(), + )); + } + let path = path.to_path(); + let username = query.username.as_deref(); + let owner_email = match username { + Some(username) => Some( + resolve_username_to_email(&w_id, username, &db) + .await? + .ok_or_else(|| { + Error::NotFound(format!("no user with username {username} in workspace")) + })?, + ), + None => None, + }; + + let mut tx = db.begin().await?; + let deleted = sqlx::query_scalar!( + r#"DELETE FROM draft + WHERE workspace_id = $1 AND path = $2 AND typ = $3 + AND ($4::text IS NULL OR email = $4) + RETURNING 1 as "one!""#, + &w_id, + path, + kind as UserDraftItemKind, + owner_email.as_deref(), + ) + .fetch_all(&mut *tx) + .await? + .len(); + if deleted == 0 { + return Err(Error::NotFound(match username { + Some(username) => format!("no draft for {username} at {path}"), + None => format!("no draft at {path}"), + })); + } + audit_log( + &mut *tx, + &authed, + "drafts.delete_for_user", + ActionKind::Delete, + &w_id, + Some(path), + Some( + [ + ("kind", kind.as_str()), + ("username", username.unwrap_or("*")), + ] + .into(), + ), + ) + .await?; + tx.commit().await?; + Ok(match username { + Some(username) => format!("Deleted {username}'s draft at {path}"), + None => format!("Deleted {deleted} draft(s) at {path}"), + }) +} + /// For variable-kind drafts with `variable.is_secret == true`, encrypt /// `variable.value` with the workspace crypt key and mark it /// `$encrypted:` so the secret never persists in plaintext at rest. @@ -1161,14 +1270,21 @@ pub(crate) async fn require_can_write_path( if authed.is_admin { return Ok(()); } - // Operators are read-only and never WRITE drafts, except a flow draft where the workspace - // granted the builder right: the kind has to be checked, or the right would open drafts of - // kinds it says nothing about. Read access is deliberately asymmetric: + // Operators are read-only and never WRITE drafts, except of a kind the workspace granted them + // the matching builder right for: a flows-only workspace must not get raw-app drafts through + // here. Read access is deliberately asymmetric: // `require_can_read_path` has no operator block, so an operator can still READ a draft they // can read via `/drafts/get`, mirroring their read access to deployed content. Intended. if authed.is_operator { - let granted = - matches!(kind, UserDraftItemKind::Flow) && operator_can_build_flows(db, w_id).await?; + let allowed = match kind { + UserDraftItemKind::Flow => Some(BuilderKind::Flows), + UserDraftItemKind::RawApp => Some(BuilderKind::Apps), + _ => None, + }; + let granted = match allowed { + Some(kind) => operator_builder_rights(db, w_id).await?.has(kind), + None => false, + }; if !granted { return Err(Error::PermissionDenied( "operators cannot save drafts".to_string(), diff --git a/backend/windmill-api/src/job_helpers_oss.rs b/backend/windmill-api/src/job_helpers_oss.rs index eb8986a986..0d6f20cdfa 100644 --- a/backend/windmill-api/src/job_helpers_oss.rs +++ b/backend/windmill-api/src/job_helpers_oss.rs @@ -263,6 +263,10 @@ mod app_s3_display_stubs { pub search_col: Option, pub search_term: Option, pub storage: Option, + pub s3_resource_path: Option, + pub csv_separator: Option, + pub csv_has_header: Option, + pub filters: Option, } #[derive(Serialize)] @@ -278,7 +282,10 @@ mod app_s3_display_stubs { pub search_col: Option, pub search_term: Option, pub storage: Option, + pub s3_resource_path: Option, pub csv_separator: Option, + pub csv_has_header: Option, + pub filters: Option, } pub async fn load_file_metadata_internal( @@ -308,6 +315,7 @@ mod app_s3_display_stubs { pub async fn load_table_count_internal( _authed: OptJobAuthed, _db: &DB, + _user_db: Option, _w_id: &str, _file_key: String, _query: LoadCountQuery, @@ -320,6 +328,7 @@ mod app_s3_display_stubs { pub async fn load_preview_internal( _authed: OptJobAuthed, _db: &DB, + _user_db: Option, _w_id: &str, _file_key: String, _query: LoadPreviewQuery, diff --git a/backend/windmill-api/src/jobs.rs b/backend/windmill-api/src/jobs.rs index 785c6ff935..2b93c6daeb 100644 --- a/backend/windmill-api/src/jobs.rs +++ b/backend/windmill-api/src/jobs.rs @@ -19,7 +19,7 @@ use serde_json::value::RawValue; use serde_json::Value; use sha2::{Digest, Sha256}; use std::borrow::Cow; -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use std::str::FromStr; use std::sync::Arc; use std::time::Instant; @@ -55,7 +55,8 @@ use windmill_common::workspace_dependencies::{ RawWorkspaceDependencies, MIN_VERSION_WORKSPACE_DEPENDENCIES, }; use windmill_common::workspaces::{ - check_operator_can_build_flows, check_user_against_rule, ProtectionRuleKind, RuleCheckResult, + check_operator_can_build, check_user_against_rule, BuilderKind, ProtectionRuleKind, + RuleCheckResult, }; use windmill_common::DYNAMIC_INPUT_CACHE; #[cfg(all(feature = "enterprise", feature = "instance_smtp"))] @@ -292,6 +293,7 @@ pub fn workspaced_service() -> Router { ) .route("/run/dynamic_select", post(run_dynamic_select)) .route("/list", get(list_jobs)) + .route("/oldest", get(get_oldest_job)) .route("/asset_dispatch_edges", get(list_asset_dispatch_edges)) .route( "/list_selected_job_groups", @@ -377,6 +379,10 @@ pub fn workspaced_service() -> Router { "/wac_approval_urls/{job_id}/{step_key}", get(get_wac_approval_urls).layer(cors.clone()), ) + .route( + "/worker_is_draining/{id}", + get(worker_is_draining).layer(cors.clone()), + ) .route( "/result_by_id/{job_id}/{node_id}", get(get_result_by_id).layer(cors.clone()), @@ -497,6 +503,28 @@ async fn get_result_by_id( view_token.as_deref(), ) .await?; + // The node is resolved up through the enclosing flows, so a read path that admitted + // `flow_id` as a step must also admit the run it is a step of. A path-scoped + // `jobs:read` does not reach this route; a `jobs:run` scope beside it does. + if windmill_api_auth::scopes::job_read_confinement(authed.scopes.as_deref()) + .is_some_and(|c| c.has_read_paths()) + { + let top = sqlx::query_scalar::<_, Uuid>( + "WITH RECURSIVE chain(id, parent_job) AS ( + SELECT id, parent_job FROM v2_job WHERE id = $1 AND workspace_id = $2 + UNION ALL + SELECT j.id, j.parent_job FROM v2_job j + JOIN chain c ON j.id = c.parent_job AND j.workspace_id = $2 + ) + SELECT id FROM chain WHERE parent_job IS NULL", + ) + .bind(flow_id) + .bind(&w_id) + .fetch_optional(&db) + .await? + .ok_or_else(|| Error::NotFound(format!("Job {flow_id} not found")))?; + require_job_within_read_scope(&db, &authed, &w_id, &top).await?; + } let res = windmill_queue::get_result_by_id(db.clone(), w_id.clone(), flow_id, node_id, json_path) @@ -590,6 +618,53 @@ async fn get_root_job( Ok(Json(res)) } +/// Whether the worker running this job has received its shutdown signal. A draining worker +/// never interrupts its job, so a long-running script polls this to exit on its own terms. +async fn worker_is_draining( + authed: ApiAuthed, + Extension(db): Extension, + Extension(user_db): Extension, + Path((w_id, id)): Path<(String, Uuid)>, +) -> windmill_common::error::JsonResult { + // A job always may ask about itself. Anyone else must be able to see the job: the + // caller's row policies on v2_job decide, narrowed by the token's tag filter. + let draining = if authed.job_id == Some(id) { + sqlx::query_scalar!( + "SELECT wp.draining FROM v2_job_queue q JOIN worker_ping wp ON wp.worker = q.worker + WHERE q.id = $1 AND q.workspace_id = $2 AND q.running AND q.started_at IS NOT NULL", + id, + w_id + ) + .fetch_optional(&db) + .await? + } else { + let tags = get_scope_tags(&authed) + .map(|tags| tags.into_iter().map(str::to_string).collect::>()); + let mut tx = user_db.begin(&authed).await?; + let draining = sqlx::query_scalar!( + "SELECT wp.draining FROM v2_job j + JOIN v2_job_queue q ON q.id = j.id + JOIN worker_ping wp ON wp.worker = q.worker + WHERE j.id = $1 AND j.workspace_id = $2 AND q.running AND q.started_at IS NOT NULL + AND ($3::text[] IS NULL OR j.tag = ANY($3))", + id, + w_id, + tags.as_deref() + ) + .fetch_optional(&mut *tx) + .await?; + tx.commit().await?; + draining + } + .unwrap_or(false); + windmill_common::feature_usage::log_feature_usage( + "worker_draining", + "check", + if draining { "draining" } else { "not_draining" }, + ); + Ok(Json(draining)) +} + async fn compute_root_job_for_flow(db: &DB, w_id: &str, job_id: Uuid) -> error::Result { let root_job = sqlx::query_scalar!( r#"SELECT COALESCE(root_job, flow_innermost_root_job, parent_job, id) as "root_job!" FROM v2_job WHERE id = $1 AND workspace_id = $2"#, @@ -620,7 +695,7 @@ async fn cancel_job_api( // right to kill someone else's run. Anonymous callers are instead confined to // anonymous-created jobs by `cancel_job`'s `require_anonymous`. if let Some(authed) = opt_authed.as_ref() { - require_job_update_read_access(&db, &user_db, authed, &w_id, &id, None).await?; + require_job_cancel_access(&db, &user_db, authed, &w_id, &id).await?; } let tx = db.begin().await?; @@ -689,7 +764,15 @@ async fn cancel_persistent_script_api( Json(CancelJob { reason }): Json, ) -> error::Result<()> { let audit_author: AuditAuthor = match opt_authed { - Some(authed) => (&authed).into(), + Some(authed) => { + let path = script_path.to_path(); + if !windmill_api_auth::scopes::job_cancel_path_confinement(authed.scopes.as_deref()) + .is_none_or(|c| windmill_api_auth::scopes::cancel_confinement_admits(&c, path)) + { + return Err(Error::NotFound(format!("Script {path} not found"))); + } + (&authed).into() + } None => { return Err(Error::BadRequest(format!( "Cancelling persistent script require to be logged in and member of {w_id}" @@ -782,7 +865,7 @@ async fn force_cancel( // caller who can only see an inner step kill a root flow hidden from them. if let Some(authed) = opt_authed.as_ref() { let target = force_cancel_target(&db, &w_id, id).await?; - require_job_update_read_access(&db, &user_db, authed, &w_id, &target, None).await?; + require_job_cancel_access(&db, &user_db, authed, &w_id, &target).await?; } let tx = db.begin().await?; @@ -1705,6 +1788,22 @@ pub(crate) async fn require_job_read_access( job_id: &Uuid, created_by: &str, view_token: Option<&str>, +) -> error::Result<()> { + require_job_access( + db, user_db, authed, w_id, job_id, created_by, view_token, true, + ) + .await +} + +async fn require_job_access( + db: &DB, + user_db: &UserDB, + authed: &ApiAuthed, + w_id: &str, + job_id: &Uuid, + created_by: &str, + view_token: Option<&str>, + read_confined: bool, ) -> error::Result<()> { // Tag scope (`if_jobs:filter_tags:`) is an orthogonal hard restriction on a // scoped token: it must never read a job outside its allowed tags, regardless of @@ -1729,10 +1828,12 @@ pub(crate) async fn require_job_read_access( } } - // A path-scoped `jobs:run` token is likewise hard-restricted to the runnables it - // may start, ahead of every grant below — the token is handed out to run one thing, - // so it must not read jobs of anything else merely because its owner could. - require_job_within_run_scope(db, authed, w_id, job_id).await?; + // A path-scoped `jobs:run` or `jobs:read` token is likewise hard-restricted to the + // runnables it names, ahead of every grant below — the token is handed out for one + // thing, so it must not read jobs of anything else merely because its owner could. + if read_confined { + require_job_within_read_scope(db, authed, w_id, job_id).await?; + } // Fast path: you can always read a job you launched. This is also load-bearing // for apps — a component job runs as the app policy's `permissioned_as`, but its @@ -1880,24 +1981,41 @@ pub(crate) async fn require_job_read_access( /// could start an app's inline-script component but not read the run back — those jobs /// are `AppScript`/`Preview` kinds that no `jobs:run` scope can name. /// -/// No-op — and no query — for every caller whose job reads are not run-confined (see -/// `job_read_run_confinement`), which is all sessions, unscoped tokens and `jobs:read` -/// tokens. -async fn require_job_within_run_scope( +/// A `jobs:read:` scope confines the same way, by path alone: the job, or one of +/// its ancestors, must be a run of a deployed script, flow or agent at a named path. +/// Only those kinds count, because a preview's `runnable_path` is whatever its caller +/// sent and would otherwise let any preview pass for a named runnable. +/// +/// No-op — and no query — for every caller whose job reads are not confined (see +/// `job_read_confinement`), which is all sessions, unscoped tokens and tokens whose +/// `jobs:read` names no path. +async fn require_job_within_read_scope( db: &DB, authed: &ApiAuthed, w_id: &str, job_id: &Uuid, ) -> error::Result<()> { let Some(confinement) = - windmill_api_auth::scopes::job_read_run_confinement(authed.scopes.as_deref()) + windmill_api_auth::scopes::job_read_confinement(authed.scopes.as_deref()) else { return Ok(()); }; + // A job whose token is restricted by `job_token_scopes` still reads its own flow run + // (step results, user state), which the route check lets through whatever the + // scopes. A read path names what else it may read, not a fence around its own run. + if let Some(own_job) = authed.job_id { + if confinement.has_read_paths() + && windmill_api_auth::auth::job_in_same_flow_run(db, own_job, *job_id).await + { + return Ok(()); + } + } // `scope_kind` is the runnable kind a `jobs:run::` scope can name, or - // NULL for a job no such scope reaches directly (previews, dependency jobs, - // flow-inlined scripts) — those are still readable as a step of a matching flow, - // through their ancestors. A `singlestepflow` wraps either a script or a flow, so it + // NULL for a job no such scope reaches directly (previews, flow-inlined scripts) — + // those are still readable as a step of a matching flow, through their ancestors. + // A dependency job's path is the one its `jobs:run:dependencies:` check + // admitted, but it is filed under `dependencies`, so it never passes for a run of + // the deployed runnable at that path. A `singlestepflow` wraps either a script or a flow, so it // projects onto the wrapped runnable the same way the batch-rerun query does. An agent // run is a preview of the one-step flow `agent_runs::agent_step_flow` builds, filed under the // agent's path (`.chat` for a chat turn); the editor's own runs of it look the same, @@ -1916,6 +2034,7 @@ async fn require_job_within_run_scope( WHEN a.agent THEN 'agents' WHEN j.kind IN ('script', 'script_hub', 'unassigned_script') THEN 'scripts' WHEN j.kind IN ('flow', 'unassigned_flow') THEN 'flows' + WHEN j.kind IN ('dependencies', 'flowdependencies') THEN 'dependencies' WHEN j.kind IN ('singlestepflow', 'unassigned_singlestepflow') THEN CASE WHEN COALESCE( (SELECT m->'value'->>'type' @@ -1941,13 +2060,7 @@ async fn require_job_within_run_scope( let in_scope = chain.iter().any( |job| match (job.runnable_path.as_deref(), job.scope_kind.as_deref()) { - (Some(runnable_path), Some(kind)) - if windmill_api_auth::scopes::run_confinement_admits( - &confinement, - kind, - runnable_path, - ) => - { + (Some(runnable_path), Some(kind)) if confinement.admits(kind, runnable_path) => { true } _ => job.launched_by_app.as_deref().is_some_and(&runs_app), @@ -1961,6 +2074,73 @@ async fn require_job_within_run_scope( } } +/// The `ids` a path-scoped `jobs:cancel:` token may cancel; all of them for +/// every caller whose cancels are not path-confined (see `job_cancel_path_confinement`). +/// +/// A job is admitted when it, or any of its `parent_job` ancestors, is a run of a +/// deployed script, flow or agent whose path the scope names: cancelling a flow's step +/// is within a scope on the flow. Only those kinds count, because a preview's +/// `runnable_path` is whatever its caller sent and would otherwise let any preview +/// impersonate an in-scope runnable. Agent runs are previews filed under the agent's +/// path, recognized the same way `require_job_within_read_scope` does. +async fn filter_jobs_within_cancel_scope( + db: &DB, + authed: &ApiAuthed, + w_id: &str, + ids: Vec, +) -> error::Result> { + let Some(confinement) = + windmill_api_auth::scopes::job_cancel_path_confinement(authed.scopes.as_deref()) + else { + return Ok(ids); + }; + let chain = sqlx::query!( + r#"WITH RECURSIVE chain(origin, id, parent_job) AS ( + SELECT id, id, parent_job FROM v2_job WHERE id = ANY($1) AND workspace_id = $2 + UNION ALL + SELECT c.origin, j.id, j.parent_job FROM v2_job j + JOIN chain c ON j.id = c.parent_job AND j.workspace_id = $2 + ) + SELECT c.origin AS "origin!", + CASE WHEN a.agent THEN regexp_replace(j.runnable_path, '\.chat$', '') + ELSE j.runnable_path END AS "runnable_path!" + FROM chain c JOIN v2_job j ON j.id = c.id, + LATERAL (SELECT j.kind = 'flowpreview' + AND j.raw_flow->'modules'->1 IS NULL + AND j.raw_flow->'modules'->0->>'id' = '__wm_agent_root' AS agent) a + WHERE j.workspace_id = $2 AND j.runnable_path IS NOT NULL + AND (a.agent OR j.kind IN ('script', 'script_hub', 'unassigned_script', 'flow', + 'unassigned_flow', 'singlestepflow', 'unassigned_singlestepflow'))"#, + &ids, + w_id, + ) + .fetch_all(db) + .await?; + let admitted: HashSet = chain + .into_iter() + .filter(|r| { + windmill_api_auth::scopes::cancel_confinement_admits(&confinement, &r.runnable_path) + }) + .map(|r| r.origin) + .collect(); + Ok(ids.into_iter().filter(|id| admitted.contains(id)).collect()) +} + +async fn require_job_within_cancel_scope( + db: &DB, + authed: &ApiAuthed, + w_id: &str, + job_id: Uuid, +) -> error::Result<()> { + if filter_jobs_within_cancel_scope(db, authed, w_id, vec![job_id]) + .await? + .is_empty() + { + return Err(Error::NotFound(format!("Job {job_id} not found"))); + } + Ok(()) +} + /// Self + every `parent_job` ancestor (intermediate sub-flows up to the top-level /// root) of `job_id`, resolved via the root DB (flow lineage is not sensitive). /// Falls back to `[job_id]` if the row is absent so callers still run their probe. @@ -2156,6 +2336,41 @@ async fn require_job_update_read_access( require_job_read_access(db, user_db, authed, w_id, job_id, &created_by, view_token).await } +/// The per-job check of the cancel routes. A `jobs:cancel` grant stands on its own, like +/// `jobs:write`: its paths confine it (`filter_jobs_within_cancel_scope`), not the +/// token's `jobs:run` scopes, which bound what a run token may read. Intersecting the two +/// would not hold anyway, since the token can mint itself a child holding only the +/// cancel scope. +async fn require_job_cancel_access( + db: &DB, + user_db: &UserDB, + authed: &ApiAuthed, + w_id: &str, + job_id: &Uuid, +) -> error::Result<()> { + let created_by = sqlx::query_scalar!( + "SELECT created_by FROM v2_job WHERE id = $1 AND workspace_id = $2", + job_id, + w_id, + ) + .fetch_optional(db) + .await? + .ok_or_else(|| Error::NotFound(format!("Job {job_id} not found")))?; + let read_confined = !windmill_api_auth::scopes::has_job_cancel_grant(authed.scopes.as_deref()); + require_job_access( + db, + user_db, + authed, + w_id, + job_id, + &created_by, + None, + read_confined, + ) + .await?; + require_job_within_cancel_scope(db, authed, w_id, *job_id).await +} + /// Whether a validated approval token should grant the job-read bypass. The token alone /// is sufficient unless the current approval step has `user_auth_required`, in which case /// only an authorized approver may read the job (and thus its args/flow inputs). @@ -2286,7 +2501,7 @@ async fn get_job( // an approval link for a job must not let a scoped token read one outside the // runnables it may start. if let Some(authed) = opt_authed.as_ref() { - require_job_within_run_scope(&db, authed, &w_id, &id).await?; + require_job_within_read_scope(&db, authed, &w_id, &id).await?; } } else { require_opt_authed_job_read_access( @@ -3204,6 +3419,7 @@ fn flow_tree_entry_label( "forloopflow" => " forloop", "whileloopflow" => " whileloop", "aiagent" => " ai-agent", + "aidecision" => " ai-decision", _ => "", }; @@ -4260,6 +4476,17 @@ struct ListableQueuedJob { pub workspace_id: String, } +/// Narrows a job list to what a path-scoped `jobs:read:` token may read: the +/// top-level runs of the runnables it names. A step is read by id, under its flow. No-op +/// for every caller whose lists are not path-filtered (see `job_list_path_filter`). +fn confine_list_to_read_scope(sqlb: &mut SqlBuilder, authed: &ApiAuthed) { + if let windmill_api_auth::ScopePathFilter::Restricted { exact, prefix } = + windmill_api_auth::scopes::job_list_path_filter(authed.scopes.as_deref()) + { + and_where_root_run_at_paths(sqlb, &exact, &prefix); + } +} + async fn list_queue_jobs( authed: ApiAuthed, Extension(user_db): Extension, @@ -4281,7 +4508,7 @@ async fn list_queue_jobs( "null as args" }; - let sql = list_queue_jobs_query( + let mut sqlb = list_queue_jobs_query( &w_id, &lq, &[ @@ -4311,8 +4538,9 @@ async fn list_queue_jobs( pagination, false, get_scope_tags(&authed), - ) - .sql()?; + ); + confine_list_to_read_scope(&mut sqlb, &authed); + let sql = sqlb.sql()?; let mut tx = user_db.begin(&authed).await?; let jobs = sqlx::query_as::<_, ListableQueuedJob>(&sql) .fetch_all(&mut *tx) @@ -4372,14 +4600,40 @@ async fn cancel_selection( let force_cancel = query.force_cancel.unwrap_or(false); let mut cancelled = Vec::new(); for (workspace_id, ids) in jobs_by_workspace { - let Json(mut w_cancelled) = cancel_jobs( - ids, - &db, - authed.username.as_str(), - workspace_id.as_str(), - force_cancel, - ) - .await?; + let ids = + if windmill_api_auth::scopes::job_cancel_path_confinement(authed.scopes.as_deref()) + .is_some() + { + // Checked on the job the cancel actually kills: a force cancel reaches the + // highest queued ancestor. + let mut targets = Vec::with_capacity(ids.len()); + for id in ids { + let target = if force_cancel { + force_cancel_target(&db, &workspace_id, id).await? + } else { + id + }; + targets.push((id, target)); + } + let admitted: HashSet = filter_jobs_within_cancel_scope( + &db, + &authed, + &workspace_id, + targets.iter().map(|(_, t)| *t).collect(), + ) + .await? + .into_iter() + .collect(); + targets + .into_iter() + .filter(|(_, t)| admitted.contains(t)) + .map(|(id, _)| id) + .collect() + } else { + ids + }; + let Json(mut w_cancelled) = + cancel_jobs(ids, &db, &authed, workspace_id.as_str(), force_cancel).await?; cancelled.append(&mut w_cancelled); } @@ -4567,6 +4821,41 @@ async fn count_completed_jobs( )) } +#[derive(Deserialize)] +struct OldestJobQuery { + all_workspaces: Option, +} + +#[derive(Serialize)] +struct OldestJob { + #[serde(skip_serializing_if = "Option::is_none")] + created_at: Option>, +} + +/// Where the job history starts, whatever the caller may see of it: a client scanning the +/// runs backwards in time windows needs a bound that row-level visibility cannot make slow. +async fn get_oldest_job( + Extension(db): Extension, + Path(w_id): Path, + Query(q): Query, +) -> error::JsonResult { + let created_at = if w_id == "admins" && q.all_workspaces.unwrap_or(false) { + // One index probe per workspace: a plain min() over the table has no index to walk. + sqlx::query_scalar( + "SELECT min(o.created_at) FROM workspace w CROSS JOIN LATERAL \ + (SELECT min(created_at) AS created_at FROM v2_job WHERE workspace_id = w.id) o", + ) + .fetch_one(&db) + .await? + } else { + sqlx::query_scalar("SELECT min(created_at) FROM v2_job WHERE workspace_id = $1") + .bind(&w_id) + .fetch_one(&db) + .await? + }; + Ok(Json(OldestJob { created_at })) +} + lazy_static::lazy_static! { /// 0 keeps the connection-wide statement_timeout. static ref LIST_JOBS_STATEMENT_TIMEOUT_SECS: u64 = @@ -4666,7 +4955,7 @@ async fn list_jobs( } let sqlc = if lq.running.is_none() { - Some(list_completed_jobs_query( + let mut sqlc = list_completed_jobs_query( &w_id, Some(per_page), 0, @@ -4674,7 +4963,9 @@ async fn list_jobs( cj_fields_ref, true, get_scope_tags(&authed), - )) + ); + confine_list_to_read_scope(&mut sqlc, &authed); + Some(sqlc) } else { None }; @@ -4701,6 +4992,7 @@ async fn list_jobs( true, get_scope_tags(&authed), ); + confine_list_to_read_scope(&mut sqlq, &authed); if let Some(sqlc) = sqlc { format!("{} UNION ALL {}", &sqlq.subquery()?, &sqlc.subquery()?,) @@ -5124,6 +5416,11 @@ async fn get_approval_info( "Must be logged in or provide a valid approval token".to_string(), )); } + // Nothing below goes through `require_job_read_access`, and an approval token is no + // way around a scoped token's confinement. + if let Some(authed) = opt_authed.as_ref() { + require_job_within_read_scope(&db, authed, &w_id, &job_id).await?; + } // Fetch job info #[derive(sqlx::FromRow)] @@ -5356,8 +5653,17 @@ async fn get_approval_info( // Possession of view rights over this approval is sufficient to mint a // share-read-link token for the flow: it only grants read (no resume), and only to // an authenticated workspace member, so it never widens what the approver can do. - let hmac = generate_view_token(&w_id, row.id, VIEW_TOKEN_DOMAIN, &db).await?; - let view_token = Some(format!("{}.{hmac}", row.id)); + // Not for a token whose reads are confined: the link would outlive it and serve any + // member, which is why `job_view_token` is refused to it too. + let confined = opt_authed.as_ref().is_some_and(|authed| { + windmill_api_auth::scopes::job_read_confinement(authed.scopes.as_deref()).is_some() + }); + let view_token = if confined { + None + } else { + let hmac = generate_view_token(&w_id, row.id, VIEW_TOKEN_DOMAIN, &db).await?; + Some(format!("{}.{hmac}", row.id)) + }; Ok(Json(ApprovalInfo { flow_id: row.id, @@ -5819,7 +6125,7 @@ pub async fn get_suspended_job_flow( // resume secret must not let a scoped token read a flow it may not run. Anonymous // approvers are unaffected. if let Some(authed) = authed.as_ref() { - require_job_within_run_scope(&db, authed, &w_id, &flow_id).await?; + require_job_within_read_scope(&db, authed, &w_id, &flow_id).await?; } let flow = GetQuery::new() @@ -6003,7 +6309,7 @@ pub async fn get_flow_user_state( // Reachable by a `jobs:run` token (it is one of the by-id routes a run needs), so // apply the same run-scope confinement as the other single-job reads. RLS below // still governs which jobs the owner's identity can see at all. - require_job_within_run_scope(&db, &authed, &w_id, &job_id).await?; + require_job_within_read_scope(&db, &authed, &w_id, &job_id).await?; let mut tx = user_db.begin(&authed).await?; let r = sqlx::query_scalar!( r#" @@ -7193,6 +7499,7 @@ pub async fn restart_flow( let tx = PushIsolationLevel::Isolated(user_db, authed.clone().into()); + let scope_ceiling = windmill_api_auth::caller_scope_ceiling(&db, &authed).await?; let (uuid, tx) = push( &db, tx, @@ -7230,6 +7537,7 @@ pub async fn restart_flow( None, authed.trigger_or_fallback(None), run_query.suspended_mode, + scope_ceiling.as_deref(), ) .await?; tx.commit().await?; @@ -7367,7 +7675,11 @@ pub async fn run_workflow_as_code( extra.insert(ENTRYPOINT_OVERRIDE.to_string(), to_raw_value(&entrypoint)); let args = PushArgs { args: &task.args.unwrap_or_else(HashMap::new), extra: Some(extra) }; check_tag_available_for_workspace(&db, &w_id, &run_query.tag, &args, &authed).await?; - check_scopes(&authed, || format!("jobs:run"))?; + // A job running its own task is its runtime, not a new run: a restricted job token keeps + // it (`is_own_job_runtime_route`), and the task inherits that job's restriction at push. + if authed.job_id != Some(job_id) { + check_scopes(&authed, || format!("jobs:run"))?; + } // The task becomes a child of `job_id`, runs its code and writes into its flow status, so // only that job itself (the SDK's `task` wrapper, on its `WM_TOKEN`) or an admin may push it. if authed.job_id != Some(job_id) && !authed.is_admin { @@ -7512,6 +7824,8 @@ pub async fn run_workflow_as_code( ) }; + let end_user_email = run_end_user_email(&db, &w_id, &authed, None).await?; + let scope_ceiling = windmill_api_auth::caller_scope_ceiling(&db, &authed).await?; let (uuid, mut tx) = push( &db, tx, @@ -7539,9 +7853,10 @@ pub async fn run_workflow_as_code( None, push_authed.as_ref(), false, + end_user_email, None, None, - None, + scope_ceiling.as_deref(), ) .await?; @@ -7825,6 +8140,8 @@ pub async fn run_wait_result_job_by_path_get( ) }; + let end_user_email = run_end_user_email(&db, &w_id, &authed, None).await?; + let scope_ceiling = windmill_api_auth::caller_scope_ceiling(&db, &authed).await?; let (uuid, tx) = push( &db, tx, @@ -7852,9 +8169,10 @@ pub async fn run_wait_result_job_by_path_get( None, push_authed.as_ref(), false, - None, + end_user_email, authed.trigger_or_fallback(None), run_query.suspended_mode, + scope_ceiling.as_deref(), ) .await?; tx.commit().await?; @@ -7973,6 +8291,8 @@ pub async fn run_wait_result_script_by_path_internal( ) }; + let end_user_email = run_end_user_email(&db, &w_id, &authed, None).await?; + let scope_ceiling = windmill_api_auth::caller_scope_ceiling(&db, &authed).await?; let (uuid, tx) = push( &db, tx, @@ -8000,9 +8320,10 @@ pub async fn run_wait_result_script_by_path_internal( None, push_authed.as_ref(), false, - None, + end_user_email, authed.trigger_or_fallback(None), run_query.suspended_mode, + scope_ceiling.as_deref(), ) .await?; tx.commit().await?; @@ -8055,6 +8376,7 @@ pub async fn run_wait_result_script_by_hash( timeout, has_preprocessor, labels, + job_token_scopes, runnable_settings: ScriptRunnableSettingsInline { concurrency_settings, debouncing_settings }, .. @@ -8088,6 +8410,8 @@ pub async fn run_wait_result_script_by_hash( ) }; + let end_user_email = run_end_user_email(&db, &w_id, &authed, None).await?; + let scope_ceiling = windmill_api_auth::caller_scope_ceiling(&db, &authed).await?; let (uuid, tx) = push( &db, tx, @@ -8105,6 +8429,7 @@ pub async fn run_wait_result_script_by_hash( apply_preprocessor: !run_query.skip_preprocessor.unwrap_or(false) && has_preprocessor.unwrap_or(false), labels, + job_token_scopes, }, push_args, authed.display_username(), @@ -8128,9 +8453,10 @@ pub async fn run_wait_result_script_by_hash( None, push_authed.as_ref(), false, - None, + end_user_email, authed.trigger_or_fallback(None), run_query.suspended_mode, + scope_ceiling.as_deref(), ) .await?; tx.commit().await?; @@ -8268,6 +8594,11 @@ pub async fn stream_job( run_query: RunJobQuery, is_get: bool, ) -> error::Result { + if run_query.retry.is_some() { + return Err(error::Error::BadRequest( + "retry is only supported when running a script asynchronously".to_string(), + )); + } let args = if is_get { let payload_as_args = run_query.payload_as_args()?; @@ -8650,6 +8981,8 @@ async fn run_preview_script( let push_args = PushArgs { extra, args: &preview_args }; check_tag_available_for_workspace(&db, &w_id, &tag, &push_args, &authed).await?; + let scope_ceiling = windmill_api_auth::caller_scope_ceiling(&db, &authed).await?; + let end_user_email = run_end_user_email(&db, &w_id, &authed, None).await?; let (uuid, tx) = push( &db, tx, @@ -8697,9 +9030,10 @@ async fn run_preview_script( None, Some(&authed.clone().into()), false, - None, + end_user_email, authed.trigger_or_fallback(None), None, + scope_ceiling.as_deref(), ) .await?; tx.commit().await?; @@ -8735,6 +9069,7 @@ async fn run_inline_preview_script( if let Some(job_id) = job_id { register_potential_assets_on_inline_execution(job_id, &w_id, &preview); } + let end_user_email = run_end_user_email(&db, &w_id, &authed, None).await?; let utils = get_worker_internal_server_inline_utils()?; let result = utils.run_inline_preview_script.as_ref()(RunInlinePreviewScriptFnParams { content: preview.content, @@ -8745,6 +9080,7 @@ async fn run_inline_preview_script( created_by: authed.display_username().to_string(), permissioned_as: username_to_permissioned_as(&authed.username), permissioned_as_email: authed.email.clone(), + end_user_email, lang: preview.language, job_dir: "".to_string(), worker_name: "".to_string(), @@ -8784,7 +9120,8 @@ async fn run_inline_script_by_path( token, db, w_id, - InlineScriptTarget::Path(script_path.to_path().to_string()), + // Resolved to a version by `run_inline_script_inner`. + InlineScriptTarget::Path { path: script_path.to_path().to_string(), hash: 0 }, body.args, Some(user_db), ) @@ -8847,6 +9184,43 @@ async fn run_inline_script_inner( args: Option>>, user_db: Option, ) -> error::Result { + // An inline run executes with the caller's own token, so it cannot honour a script's + // `job_token_scopes`: such a script only runs as a job. A path is resolved here, once, and + // the version checked is the version run. + let (target, restricted) = match target { + InlineScriptTarget::Path { path, .. } => { + let authed_ref = authed.to_authed_ref(); + let info = get_latest_deployed_hash_for_path( + user_db + .as_ref() + .map(|db| UserDbWithAuthed { db: db.clone(), authed: &authed_ref }), + db.clone(), + &w_id, + &path, + ) + .await?; + let restricted = info.job_token_scopes.is_some(); + ( + InlineScriptTarget::Path { path, hash: info.hash }, + restricted, + ) + } + InlineScriptTarget::Hash(hash) => { + let restricted = windmill_common::get_script_info_for_hash(None, &db, &w_id, hash) + .await? + .job_token_scopes + .is_some(); + (InlineScriptTarget::Hash(hash), restricted) + } + }; + if restricted { + return Err(Error::BadRequest( + "This script restricts its job token (job_token_scopes) and cannot be run inline; \ + run it as a job instead" + .to_string(), + )); + } + let end_user_email = run_end_user_email(&db, &w_id, &authed, None).await?; let utils = get_worker_internal_server_inline_utils()?; let authed_owned: windmill_common::db::Authed = authed.clone().into(); let result = utils.run_inline_script.as_ref()(RunInlineScriptFnParams { @@ -8858,6 +9232,7 @@ async fn run_inline_script_inner( created_by: authed.display_username().to_string(), permissioned_as: username_to_permissioned_as(&authed.username), permissioned_as_email: authed.email.clone(), + end_user_email, job_dir: "".to_string(), worker_name: "".to_string(), worker_dir: "".to_string(), @@ -9033,6 +9408,8 @@ async fn run_bundle_preview_script( // tracing::info!("is_tar 1: {is_tar}"); // hmap.insert("") + let scope_ceiling = windmill_api_auth::caller_scope_ceiling(&db, &authed).await?; + let end_user_email = run_end_user_email(&db, &w_id, &authed, None).await?; let (uuid, ntx) = push( &db, ltx, @@ -9076,9 +9453,10 @@ async fn run_bundle_preview_script( None, Some(&authed.clone().into()), false, - None, + end_user_email, authed.trigger_or_fallback(None), None, + scope_ceiling.as_deref(), ) .await?; job_id = Some(uuid); @@ -9155,13 +9533,23 @@ pub struct RunDependenciesResponse { pub dependencies: String, } +/// The path comes from the request body: one carrying scope syntax (`,`, `:`, `*`) could +/// put a granted path beside an ungranted one, so it needs a grant covering every path. +fn check_dependencies_scope(authed: &ApiAuthed, path: &str) -> error::Result<()> { + if windmill_common::auth::is_scope_literal_path(path) { + check_scopes(authed, || format!("jobs:run:dependencies:{path}")) + } else { + check_scopes(authed, || "jobs:run:dependencies".to_string()) + } +} + async fn push_dependencies_job( authed: &ApiAuthed, db: &DB, w_id: &str, req: RunDependenciesRequest, ) -> error::Result { - check_scopes(authed, || format!("jobs:run"))?; + check_dependencies_scope(authed, &req.entrypoint)?; if authed.is_operator { return Err(error::Error::NotAuthorized( "Operators cannot run dependencies jobs for security reasons".to_string(), @@ -9198,6 +9586,7 @@ async fn push_dependencies_job( req.temp_script_refs .map(|v| hm.insert("temp_script_refs".to_owned(), to_raw_value(&v))); + let scope_ceiling = windmill_api_auth::caller_scope_ceiling(db, authed).await?; let (uuid, tx) = push( db, PushIsolationLevel::IsolatedRoot(db.clone()), @@ -9232,6 +9621,7 @@ async fn push_dependencies_job( None, None, None, + scope_ceiling.as_deref(), ) .await?; tx.commit().await?; @@ -9282,8 +9672,15 @@ async fn push_flow_dependencies_job( w_id: &str, req: RunFlowDependenciesRequest, ) -> error::Result { - check_scopes(authed, || format!("jobs:run"))?; - check_operator_can_build_flows(db, w_id, authed.is_operator, "run dependencies jobs").await?; + check_dependencies_scope(authed, &req.path)?; + check_operator_can_build( + db, + w_id, + authed.is_operator, + BuilderKind::Flows, + "run dependencies jobs", + ) + .await?; // The dependency job locks whatever inline code this request carries, on a worker. A // composition-only flow has none, so validating here costs a builder nothing and keeps the // lock step from becoming the way to run code the write path refuses. @@ -9330,6 +9727,7 @@ async fn push_flow_dependencies_job( req.temp_script_refs .map(|v| args_map.insert("temp_script_refs".to_string(), to_raw_value(&v))); + let scope_ceiling = windmill_api_auth::caller_scope_ceiling(db, authed).await?; let (uuid, tx) = push( db, PushIsolationLevel::IsolatedRoot(db.clone()), @@ -9360,6 +9758,7 @@ async fn push_flow_dependencies_job( None, None, None, + scope_ceiling.as_deref(), ) .await?; tx.commit().await?; @@ -9414,6 +9813,7 @@ async fn add_batch_jobs( ) -> error::JsonResult> { require_super_admin(&db, &authed).await?; + let mut job_token_scopes: Option> = None; let ( hash, path, @@ -9445,11 +9845,13 @@ async fn add_batch_jobs( dedicated_worker, timeout, runnable_settings, + job_token_scopes: script_job_token_scopes, .. // TODO: consider on_behalf_of_email and created_by for batch jobs } = get_latest_deployed_hash_for_path(Some(db_authed), db.clone(), &w_id, &path) .await? .prefetch_cached(&db) .await?; + job_token_scopes = script_job_token_scopes; ( Some(script_hash), Some(path), @@ -9496,7 +9898,7 @@ async fn add_batch_jobs( } else if let Some(path) = batch_info.path { let mut tx = user_db.clone().begin(&authed).await?; let value_json = sqlx::query!( - "SELECT coalesce(flow_version_lite.value, flow_version.value) as \"value!: sqlx::types::Json>\" FROM flow + "SELECT coalesce(flow_version_lite.value, flow_version.value) as \"value!: sqlx::types::Json>\", flow.job_token_scopes FROM flow LEFT JOIN flow_version ON flow_version.id = flow.versions[array_upper(flow.versions, 1)] LEFT JOIN flow_version_lite @@ -9507,6 +9909,7 @@ async fn add_batch_jobs( .fetch_optional(&mut *tx) .await? .ok_or_else(|| Error::internal_err(format!("not found flow at path {:?}", path)))?; + job_token_scopes = value_json.job_token_scopes; let value = serde_json::from_str::(value_json.value.get()).map_err(|err| { Error::internal_err(format!( @@ -9519,6 +9922,11 @@ async fn add_batch_jobs( "Path is required if no value is not provided" ))? }; + if windmill_common::scopes::validate_flow_step_job_token_scopes(&value)? { + windmill_common::min_version::MIN_VERSION_SUPPORTS_JOB_TOKEN_SCOPES + .assert() + .await?; + } add_virtual_items_if_necessary(&mut value.modules); let flow_status = FlowStatus::new(&value); ( @@ -9562,7 +9970,8 @@ async fn add_batch_jobs( let language = language.unwrap_or(ScriptLang::Deno); let tag = if let Some(dedicated_worker) = dedicated_worker { - if dedicated_worker && path.is_some() { + // Same rule as `push`: a dedicated worker would run it with its own unscoped token. + if dedicated_worker && path.is_some() && job_token_scopes.is_none() { windmill_common::worker::dedicated_worker_tag(&w_id, &path.clone().unwrap()) } else { format!("{}", language.as_str()) @@ -9630,8 +10039,8 @@ async fn add_batch_jobs( .await?; sqlx::query!( - "INSERT INTO job_perms (job_id, email, username, is_admin, is_operator, folders, groups, workspace_id) - SELECT unnest($1::uuid[]), $2, $3, $4, $5, $6, $7, $8", + "INSERT INTO job_perms (job_id, email, username, is_admin, is_operator, folders, groups, workspace_id, job_token_scopes) + SELECT unnest($1::uuid[]), $2, $3, $4, $5, $6, $7, $8, $9", &uuids, authed.email, authed.username, @@ -9640,6 +10049,7 @@ async fn add_batch_jobs( &[], &[], w_id, + job_token_scopes.as_deref() as Option<&[String]>, ) .execute(&mut *tx) .await?; @@ -9685,11 +10095,25 @@ async fn run_preview_flow_job( Query(run_query): Query, Json(raw_flow): Json, ) -> error::Result<(StatusCode, String)> { - check_operator_can_build_flows(&db, &w_id, authed.is_operator, "run preview jobs").await?; + check_operator_can_build( + &db, + &w_id, + authed.is_operator, + BuilderKind::Flows, + "run preview jobs", + ) + .await?; // Flow preview runs an arbitrary, request-supplied flow definition; require the broad // jobs:run scope so a narrowly-scoped token cannot escape its scope. See run_preview_script. check_scopes(&authed, || format!("jobs:run"))?; require_path_read_access_for_preview(&authed, &raw_flow.path)?; + // Step restrictions apply to a preview as they do to a deployed run, so they are checked + // the same way: an invalid entry or an older worker would leave a step unrestricted. + if windmill_common::scopes::validate_flow_step_job_token_scopes(&raw_flow.value)? { + windmill_common::min_version::MIN_VERSION_SUPPORTS_JOB_TOKEN_SCOPES + .assert() + .await?; + } // A builder must be able to test what it composes, but the submitted value is not the stored // one: without this the preview is a way to run inline code the write path refuses. if authed.is_operator { @@ -9732,6 +10156,8 @@ async fn run_preview_flow_job( check_tag_available_for_workspace(&db, &w_id, &tag, &PushArgs::from(&flow_args), &authed) .await?; + let scope_ceiling = windmill_api_auth::caller_scope_ceiling(&db, &authed).await?; + let end_user_email = run_end_user_email(&db, &w_id, &authed, None).await?; let (uuid, mut tx) = push( &db, tx, @@ -9763,9 +10189,10 @@ async fn run_preview_flow_job( None, Some(&authed.clone().into()), false, - None, + end_user_email, authed.trigger_or_fallback(None), None, + scope_ceiling.as_deref(), ) .await?; @@ -9989,6 +10416,7 @@ async fn run_dynamic_select( let scheduled_for = run_query.get_scheduled_for(&db).await?; let tx = PushIsolationLevel::Isolated(user_db.clone(), authed.clone().into()); + let scope_ceiling = windmill_api_auth::caller_scope_ceiling(&db, &authed).await?; let (uuid, tx) = push( &db, tx, @@ -10035,6 +10463,7 @@ async fn run_dynamic_select( None, authed.trigger_or_fallback(None), None, + scope_ceiling.as_deref(), ) .await?; tx.commit().await?; @@ -10110,6 +10539,7 @@ pub async fn run_job_by_hash_inner( delete_after_use, delete_after_secs, labels, + job_token_scopes, .. } = script_info; @@ -10142,24 +10572,30 @@ pub async fn run_job_by_hash_inner( ) }; + let job_payload = JobPayload::ScriptHash { + hash: ScriptHash(hash), + path: path, + concurrency_settings, + debouncing_settings, + cache_ttl, + cache_ignore_s3_path, + language, + dedicated_worker, + priority, + apply_preprocessor: !run_query.skip_preprocessor.unwrap_or(false) + && has_preprocessor.unwrap_or(false), + labels, + job_token_scopes, + }; + let job_payload = with_run_retry(&run_query, job_payload, &push_args, &tag)?; + + let end_user_email = run_end_user_email(&db, &w_id, &authed, trigger.as_ref()).await?; + let scope_ceiling = windmill_api_auth::caller_scope_ceiling(&db, &authed).await?; let (uuid, tx) = push( &db, tx, &w_id, - JobPayload::ScriptHash { - hash: ScriptHash(hash), - path: path, - concurrency_settings, - debouncing_settings, - cache_ttl, - cache_ignore_s3_path, - language, - dedicated_worker, - priority, - apply_preprocessor: !run_query.skip_preprocessor.unwrap_or(false) - && has_preprocessor.unwrap_or(false), - labels, - }, + job_payload, push_args, authed.display_username(), email, @@ -10182,9 +10618,10 @@ pub async fn run_job_by_hash_inner( None, push_authed.as_ref(), false, - None, + end_user_email, authed.trigger_or_fallback(trigger), run_query.suspended_mode, + scope_ceiling.as_deref(), ) .await?; tx.commit().await?; @@ -11145,7 +11582,7 @@ async fn list_completed_jobs( "null as args" }; - let sql = list_completed_jobs_query( + let mut sqlb = list_completed_jobs_query( &w_id, Some(per_page), offset, @@ -11185,8 +11622,9 @@ async fn list_completed_jobs( ], false, get_scope_tags(&authed), - ) - .sql()?; + ); + confine_list_to_read_scope(&mut sqlb, &authed); + let sql = sqlb.sql()?; let mut tx = user_db.begin(&authed).await?; let jobs = sqlx::query_as::<_, ListableCompletedJob>(&sql) .fetch_all(&mut *tx) @@ -11365,7 +11803,7 @@ async fn get_completed_job_result( // confinement that gate carries — re-apply it, as `get_job` does for the // approval token. Anonymous approval access is untouched. if let Some(authed) = opt_authed.as_ref() { - require_job_within_run_scope(&db, authed, &w_id, &id).await?; + require_job_within_read_scope(&db, authed, &w_id, &id).await?; } } else { require_opt_authed_job_read_access( diff --git a/backend/windmill-api/src/lib.rs b/backend/windmill-api/src/lib.rs index 1cd2b4101c..92291bd1ea 100644 --- a/backend/windmill-api/src/lib.rs +++ b/backend/windmill-api/src/lib.rs @@ -149,6 +149,8 @@ pub mod oauth2_oss; #[cfg(feature = "private")] pub mod oidc_ee; mod oidc_oss; +#[cfg(all(feature = "enterprise", feature = "openidconnect", feature = "private"))] +pub mod oidc_signer_tools_ee; mod path_autocomplete; mod raw_apps; mod resources; diff --git a/backend/windmill-api/src/live_migrations.rs b/backend/windmill-api/src/live_migrations.rs index 334ee340a8..a620807902 100644 --- a/backend/windmill-api/src/live_migrations.rs +++ b/backend/windmill-api/src/live_migrations.rs @@ -8,7 +8,9 @@ use sqlx::{PgConnection, Postgres}; use tokio::task::JoinHandle; -use windmill_common::{db::DB, error::Error}; +use windmill_common::{ + db::DB, error::Error, global_settings::SYNC_HUB_RESOURCE_TYPES_DAILY_SETTING, +}; use crate::db::CustomMigrator; use sqlx::migrate::Migrate; @@ -160,7 +162,11 @@ async fn run_background_migrations(db: &DB) -> Result<(), Error> { return Ok(()); } - for step in [AUDIT_OPERATION_INDEX, RETIRE_LEGACY_AUDIT] { + for step in [ + RETIRE_LEGACY_HUB_SYNC, + AUDIT_OPERATION_INDEX, + RETIRE_LEGACY_AUDIT, + ] { if background_migration_done(&mut conn, step).await? { continue; } @@ -171,6 +177,7 @@ async fn run_background_migrations(db: &DB) -> Result<(), Error> { let run = match step { AUDIT_OPERATION_INDEX => create_audit_operation_index(&mut conn).await, RETIRE_LEGACY_AUDIT => retire_legacy_audit_table(&mut conn).await, + RETIRE_LEGACY_HUB_SYNC => retire_legacy_hub_sync(&mut conn).await, _ => unreachable!("background migration {step} has no step function"), }; match run { @@ -192,6 +199,7 @@ async fn run_background_migrations(db: &DB) -> Result<(), Error> { const AUDIT_OPERATION_INDEX: &str = "audit_partitioned_workspace_operation_index"; const RETIRE_LEGACY_AUDIT: &str = "retire_legacy_audit_table"; +const RETIRE_LEGACY_HUB_SYNC: &str = "retire_legacy_hub_sync_script"; // Short on purpose: a statement waiting for its lock is also a wait for every audit insert // queued behind it. @@ -371,6 +379,70 @@ async fn retire_legacy_audit_table(conn: &mut PgConnection) -> Result<(), Error> Ok(()) } +/// The hash the migrations gave `u/admin/hub_sync`; they rewrote its content in place. +const LEGACY_HUB_SYNC_HASH: i64 = -28028598712388162; + +/// `u/admin/hub_sync` and the schedules running it synced resource types before servers did +/// it themselves (`windmill_common::hub_resource_types`). Retired only while the script is +/// still the version Windmill created: a copy someone edited is theirs to keep running. +async fn retire_legacy_hub_sync(conn: &mut PgConnection) -> Result<(), Error> { + let mut tx = conn.begin().await?; + let latest: Option = sqlx::query_scalar( + "SELECT hash FROM script + WHERE workspace_id = 'admins' AND path = 'u/admin/hub_sync' AND archived = false + ORDER BY created_at DESC LIMIT 1", + ) + .fetch_optional(&mut *tx) + .await?; + if latest != Some(LEGACY_HUB_SYNC_HASH) { + return Ok(()); + } + + let schedules: Vec<(String, bool, serde_json::Value)> = sqlx::query_as( + "SELECT path, enabled, jsonb_build_object('row', to_jsonb(s)) FROM schedule s + WHERE workspace_id = 'admins' AND script_path = 'u/admin/hub_sync' AND NOT is_flow", + ) + .fetch_all(&mut *tx) + .await?; + // An enabled schedule hands over to the server's daily sync in the transaction that deletes + // it, so the instance never has both syncs running, nor neither. + if schedules.iter().any(|(_, enabled, _)| *enabled) { + let inserted = sqlx::query( + "INSERT INTO global_settings (name, value) VALUES ($1, 'true'::jsonb) + ON CONFLICT (name) DO NOTHING", + ) + .bind(SYNC_HUB_RESOURCE_TYPES_DAILY_SETTING) + .execute(&mut *tx) + .await? + .rows_affected(); + if inserted > 0 { + tracing::info!( + "u/admin/hub_sync ran on an enabled schedule: turned on {SYNC_HUB_RESOURCE_TYPES_DAILY_SETTING}" + ); + } + } + for (path, _, row) in schedules { + windmill_queue::schedule::clear_schedule(&mut tx, &path, "admins").await?; + sqlx::query("DELETE FROM schedule WHERE workspace_id = 'admins' AND path = $1") + .bind(&path) + .execute(&mut *tx) + .await?; + windmill_common::trashbin::move_to_trash( + &mut *tx, "admins", "schedule", &path, row, "system", + ) + .await?; + tracing::info!("Deleted schedule {path}, which ran u/admin/hub_sync"); + } + sqlx::query( + "UPDATE script SET archived = true WHERE workspace_id = 'admins' AND path = 'u/admin/hub_sync'", + ) + .execute(&mut *tx) + .await?; + tx.commit().await?; + tracing::info!("Archived u/admin/hub_sync: servers sync resource types themselves"); + Ok(()) +} + #[cfg(test)] mod tests { use super::*; @@ -412,4 +484,80 @@ mod tests { assert_eq!(seen, 2); Ok(()) } + + async fn insert_legacy_hub_sync_schedule(db: &DB, enabled: bool) -> anyhow::Result<()> { + sqlx::query( + "INSERT INTO schedule (workspace_id, path, edited_by, email, permissioned_as, schedule, timezone, script_path, is_flow, enabled) + VALUES ('admins', 'g/all/hub_sync', 'admin', 'admin@windmill.dev', 'u/admin', '0 0 0 * * *', 'Etc/UTC', 'u/admin/hub_sync', false, $1)", + ) + .bind(enabled) + .execute(db) + .await?; + Ok(()) + } + + async fn schedules_and_live_versions(db: &DB) -> anyhow::Result<(i64, i64)> { + Ok(sqlx::query_as( + "SELECT (SELECT count(*) FROM schedule WHERE workspace_id = 'admins'), + (SELECT count(*) FROM script WHERE path = 'u/admin/hub_sync' AND NOT archived)", + ) + .fetch_one(db) + .await?) + } + + async fn daily_sync_setting(db: &DB) -> anyhow::Result> { + Ok( + sqlx::query_scalar("SELECT value FROM global_settings WHERE name = $1") + .bind(SYNC_HUB_RESOURCE_TYPES_DAILY_SETTING) + .fetch_optional(db) + .await?, + ) + } + + #[sqlx::test(migrations = "../migrations")] + async fn retire_legacy_hub_sync_spares_an_edited_script(db: DB) -> anyhow::Result<()> { + insert_legacy_hub_sync_schedule(&db, true).await?; + sqlx::query( + "INSERT INTO script (workspace_id, hash, path, content, created_by, language, summary, description, schema, lock, created_at) + SELECT workspace_id, 42, path, content || '// edited', 'admin', language, summary, description, schema, lock, now() + interval '1 minute' + FROM script WHERE workspace_id = 'admins' AND path = 'u/admin/hub_sync'", + ) + .execute(&db) + .await?; + + retire_legacy_hub_sync(&mut *db.acquire().await?).await?; + assert_eq!(schedules_and_live_versions(&db).await?, (1, 2)); + assert_eq!(daily_sync_setting(&db).await?, None); + + // Unedited, the enabled schedule hands over to the server's daily sync. + sqlx::query("DELETE FROM script WHERE hash = 42") + .execute(&db) + .await?; + retire_legacy_hub_sync(&mut *db.acquire().await?).await?; + assert_eq!(schedules_and_live_versions(&db).await?, (0, 0)); + assert_eq!( + daily_sync_setting(&db).await?, + Some(serde_json::json!(true)) + ); + Ok(()) + } + + #[sqlx::test(migrations = "../migrations")] + async fn retire_legacy_hub_sync_leaves_the_daily_sync_off_without_an_enabled_schedule( + db: DB, + ) -> anyhow::Result<()> { + for disabled_schedule in [true, false] { + sqlx::query("UPDATE script SET archived = false WHERE path = 'u/admin/hub_sync'") + .execute(&db) + .await?; + if disabled_schedule { + insert_legacy_hub_sync_schedule(&db, false).await?; + } + + retire_legacy_hub_sync(&mut *db.acquire().await?).await?; + assert_eq!(schedules_and_live_versions(&db).await?, (0, 0)); + assert_eq!(daily_sync_setting(&db).await?, None); + } + Ok(()) + } } diff --git a/backend/windmill-api/src/mcp/auto_generated_endpoints.rs b/backend/windmill-api/src/mcp/auto_generated_endpoints.rs index 1962710578..73b1fd1792 100644 --- a/backend/windmill-api/src/mcp/auto_generated_endpoints.rs +++ b/backend/windmill-api/src/mcp/auto_generated_endpoints.rs @@ -2024,7 +2024,7 @@ You should get the schema of the script or flow before creating the schedule to }, "exponential": { "type": "object", - "description": "Retry with exponential backoff (delay doubles each time)", + "description": "Retry with exponential backoff: the n-th retry waits multiplier × seconds^n", "properties": { "attempts": { "type": "integer", @@ -2037,7 +2037,7 @@ You should get the schema of the script or flow before creating the schedule to "seconds": { "type": "integer", "minimum": 1, - "description": "Initial delay in seconds" + "description": "Base of the exponential (seconds); the n-th retry waits multiplier × seconds^n, where n counts constant retries too" }, "random_factor": { "type": "integer", @@ -2231,7 +2231,7 @@ You should get the schema of the script or flow before updating the schedule to }, "exponential": { "type": "object", - "description": "Retry with exponential backoff (delay doubles each time)", + "description": "Retry with exponential backoff: the n-th retry waits multiplier × seconds^n", "properties": { "attempts": { "type": "integer", @@ -2244,7 +2244,7 @@ You should get the schema of the script or flow before updating the schedule to "seconds": { "type": "integer", "minimum": 1, - "description": "Initial delay in seconds" + "description": "Base of the exponential (seconds); the n-th retry waits multiplier × seconds^n, where n counts constant retries too" }, "random_factor": { "type": "integer", diff --git a/backend/windmill-api/src/mcp/core.rs b/backend/windmill-api/src/mcp/core.rs index 472c7bbec9..8f3962da13 100644 --- a/backend/windmill-api/src/mcp/core.rs +++ b/backend/windmill-api/src/mcp/core.rs @@ -372,7 +372,8 @@ impl McpBackend for WindmillBackend { token: &str, workspace_id: &str, ) -> BackendResult { - self.auth_cache + let authed = self + .auth_cache .get_authed(Some(workspace_id.to_string()), token) .await .ok_or_else(|| { @@ -383,7 +384,11 @@ impl McpBackend for WindmillBackend { ), None, ) - }) + })?; + // Job token scopes never include MCP scopes, so a restricted job token gets none. + windmill_api_auth::check_job_token_scope(&authed, || "mcp:all".to_string()) + .map_err(|e| ErrorData::invalid_params(e.to_string(), None))?; + Ok(authed) } async fn runnable_list_fingerprint(&self, workspace_id: &str) -> BackendResult { diff --git a/backend/windmill-api/src/mcp_tools.rs b/backend/windmill-api/src/mcp_tools.rs index e1931873a9..1e07f374de 100644 --- a/backend/windmill-api/src/mcp_tools.rs +++ b/backend/windmill-api/src/mcp_tools.rs @@ -23,14 +23,12 @@ async fn with_deadline( what: &str, fut: impl std::future::Future>, ) -> Result { - tokio::time::timeout(MCP_DEADLINE, fut) - .await - .map_err(|_| { - Error::ExecutionErr(format!( - "MCP server did not answer within {}s ({what})", - MCP_DEADLINE.as_secs() - )) - })? + tokio::time::timeout(MCP_DEADLINE, fut).await.map_err(|_| { + Error::ExecutionErr(format!( + "MCP server did not answer within {}s ({what})", + MCP_DEADLINE.as_secs() + )) + })? } /// Connect to the MCP server described by the `mcp` resource at `path`. @@ -65,8 +63,9 @@ async fn connect_mcp_client( let resource_value = not_found_if_none(resource_value_o, "Resource", path)? .ok_or_else(|| Error::BadRequest(format!("Empty resource value for {}", path)))?; - let mcp_resource = serde_json::from_str::(resource_value.0.get()) - .map_err(|e| Error::BadRequest(format!("Failed to parse MCP resource: {}", e)))?; + let mut mcp_resource = + serde_json::from_str::(resource_value.0.get()) + .map_err(|e| Error::BadRequest(format!("Failed to parse MCP resource: {}", e)))?; #[cfg(feature = "oauth2")] { @@ -126,6 +125,15 @@ async fn connect_mcp_client( None }; + // Header secrets go through the same permissioned path as the token. + for value in mcp_resource.headers.iter_mut().flatten().map(|(_, v)| v) { + if let Some(var_path) = value.strip_prefix("$var:") { + check_scopes(authed, || format!("variables:read:{}", var_path))?; + let db_authed = DbWithOptAuthed::from_authed(authed, db.clone(), Some(user_db.clone())); + *value = get_value_internal(&db_authed, w_id, var_path, false).await?; + } + } + windmill_mcp::McpClient::from_resource(mcp_resource, token) .await .map_err(|e| Error::ExecutionErr(format!("Failed to connect to MCP server: {}", e))) diff --git a/backend/windmill-api/src/offboarding.rs b/backend/windmill-api/src/offboarding.rs index 26eb06c8e8..b7f83dfd96 100644 --- a/backend/windmill-api/src/offboarding.rs +++ b/backend/windmill-api/src/offboarding.rs @@ -891,8 +891,8 @@ async fn offboard_user_from_workspace<'c>( let flows_reassigned = sqlx::query_scalar!( r#"WITH inserted AS ( INSERT INTO flow - (workspace_id, path, summary, description, archived, extra_perms, dependency_job, tag, ws_error_handler_muted, dedicated_worker, timeout, visible_to_runner_only, on_behalf_of, on_behalf_of_email, concurrency_key, versions, value, schema, edited_by, edited_at, labels, lock_error_logs) - SELECT workspace_id, REGEXP_REPLACE(path, 'u/' || $2 || '/(.*)', $1 || '/\1'), summary, description, archived, extra_perms, dependency_job, tag, ws_error_handler_muted, dedicated_worker, timeout, visible_to_runner_only, on_behalf_of, on_behalf_of_email, concurrency_key, versions, value, schema, edited_by, edited_at, labels, lock_error_logs + (workspace_id, path, summary, description, archived, extra_perms, dependency_job, tag, ws_error_handler_muted, dedicated_worker, timeout, visible_to_runner_only, on_behalf_of, on_behalf_of_email, concurrency_key, versions, value, schema, edited_by, edited_at, labels, lock_error_logs, job_token_scopes) + SELECT workspace_id, REGEXP_REPLACE(path, 'u/' || $2 || '/(.*)', $1 || '/\1'), summary, description, archived, extra_perms, dependency_job, tag, ws_error_handler_muted, dedicated_worker, timeout, visible_to_runner_only, on_behalf_of, on_behalf_of_email, concurrency_key, versions, value, schema, edited_by, edited_at, labels, lock_error_logs, job_token_scopes FROM flow WHERE path LIKE ('u/' || $2 || '/%') AND workspace_id = $3 RETURNING 1 diff --git a/backend/windmill-api/src/runnables.rs b/backend/windmill-api/src/runnables.rs index 2cee0efe33..7e64b48a1a 100644 --- a/backend/windmill-api/src/runnables.rs +++ b/backend/windmill-api/src/runnables.rs @@ -34,7 +34,7 @@ use std::collections::HashMap; use windmill_common::{ db::UserDB, error::{Error, JsonResult}, - workspaces::operator_can_build_flows, + workspaces::{operator_builder_rights, OperatorBuilderRights}, }; use windmill_types::scripts::ScriptHash; use windmill_types::user_drafts::DraftUserRef; @@ -371,7 +371,8 @@ fn draft_branch_sql(kind: &str) -> String { "SELECT '{kind}' as kind, o.path, o.summary, o.workspace_id, '{{}}'::jsonb as extra_perms, \ false as starred, false as archived, \ true as is_draft, true as draft_only, o.draft_path, \ - json_build_array(json_build_object('username', $2::text)) as draft_users, \ + json_build_array(json_build_object('username', \ + CASE WHEN o.legacy THEN NULL ELSE $2::text END)) as draft_users, \ NULL::text[] as labels, NULL::text[] as inherited_labels, \ NULL::bool as ws_error_handler_muted, o.created_at as edited_at, \ NULL::bigint as hash, o.language, o.script_kind, o.auto_kind, \ @@ -380,6 +381,7 @@ fn draft_branch_sql(kind: &str) -> String { o.created_at as sort_time, lower(COALESCE(NULLIF(o.summary, ''), o.draft_path, o.path)) as sort_name, 0::bigint as tiebreak \ FROM ( \ SELECT DISTINCT ON (d.path) d.workspace_id, d.path, d.created_at, \ + d.email IS NULL as legacy, \ COALESCE(d.value->>'summary', '') as summary, \ NULLIF(NULLIF(d.value->>'{typed_path}', ''), d.path) as draft_path, \ {kind_cols} \ @@ -572,9 +574,14 @@ async fn list_runnables( // Draft-only rows are the caller's own work in progress: never archived, so they // have no place in the archived view, and carrying no labels of their own they are // out of scope of a label filter (as in the per-kind endpoints). Operators don't - // see other people's drafts and have none of their own, except a builder's flows. + // see other people's drafts and have none of their own, except of what they may build. + let builder = if authed.is_operator { + operator_builder_rights(&db, &w_id).await? + } else { + OperatorBuilderRights::default() + }; let include_drafts = q.include_draft_only.unwrap_or(false) - && (!authed.is_operator || operator_can_build_flows(&db, &w_id).await?) + && (!authed.is_operator || builder.any()) && !show_archived && q.label.as_ref().filter(|s| !s.is_empty()).is_none(); let draft_extras_for = |kind: &str| -> Vec { @@ -660,7 +667,10 @@ async fn list_runnables( keyset: Option<&str>, limit: Option| -> Option { - if !include_drafts || !kinds.contains(&kind) || (authed.is_operator && kind != "flow") { + if !include_drafts + || !kinds.contains(&kind) + || (authed.is_operator && !operator_drafts_kind(builder, kind)) + { return None; } // `fav` is ignored: with no favorite join there is nothing to filter on, and the @@ -988,6 +998,15 @@ async fn count_runnables_by_owner( Ok(Json(RunnableCountsResponse { counts })) } +/// Whether an operator with `builder` rights can have drafts of a `/list` kind. +fn operator_drafts_kind(builder: OperatorBuilderRights, kind: &str) -> bool { + match kind { + "flow" => builder.flows, + "app" => builder.apps, + _ => false, + } +} + /// Adds the caller's draft-only rows to `counts`, in the same shape `/list` /// returns them so a badge never disagrees with the rows behind it. /// @@ -1008,13 +1027,21 @@ async fn add_draft_counts( if !q.include_draft_only.unwrap_or(false) { return Ok(()); } - // An operator has no drafts of their own, except a builder's flows. + // An operator has no drafts of their own, except of what they may build. + let builder_kinds: Vec<&str>; let kinds: &[&str] = if !authed.is_operator { kinds - } else if kinds.contains(&"flow") && operator_can_build_flows(db, w_id).await? { - &["flow"] } else { - return Ok(()); + let builder = operator_builder_rights(db, w_id).await?; + builder_kinds = kinds + .iter() + .copied() + .filter(|k| operator_drafts_kind(builder, k)) + .collect(); + if builder_kinds.is_empty() { + return Ok(()); + } + &builder_kinds }; // $1 = workspace, $2 = the caller's email. let mut binds: Vec = vec![]; diff --git a/backend/windmill-api/src/static_assets.rs b/backend/windmill-api/src/static_assets.rs index 66dc4da9df..e469e91249 100644 --- a/backend/windmill-api/src/static_assets.rs +++ b/backend/windmill-api/src/static_assets.rs @@ -100,6 +100,21 @@ fn serve_path(path: &str, original_path: &str, query: Option<&str>) -> Response< res = res.header("X-Robots-Tag", "noindex, nofollow"); } + // The raw-app preview shell evaluates whatever js is posted to it, so it is + // only ever served opaque-origin, with the deployed app wrapper's exact flags + // (`get_raw_app_data`). Matched on the embedded file, not the request path, + // so `//ui_builder/...` is covered. A sandboxed app's editor preview loads it + // as served, framed with the same flags (`RAW_APP_SANDBOX_FLAGS` in the + // frontend); an unsandboxed one loads a same-origin blob: copy. + if path == "ui_builder/app-preview.html" { + res = res.header( + header::CONTENT_SECURITY_POLICY, + "sandbox allow-scripts allow-forms allow-popups \ + allow-popups-to-escape-sandbox allow-downloads allow-modals \ + allow-top-navigation", + ); + } + // Add Content-Security-Policy header for static assets when policy is set if !CSP_POLICY.is_empty() { if let Ok(header_value) = HeaderValue::try_from(CSP_POLICY.as_str()) { diff --git a/backend/windmill-api/src/token.rs b/backend/windmill-api/src/token.rs index a1a9206196..ab22f2d15b 100644 --- a/backend/windmill-api/src/token.rs +++ b/backend/windmill-api/src/token.rs @@ -193,13 +193,18 @@ lazy_static! { ScopeOption { value: "jobs:read".to_string(), label: "Read".to_string(), - requires_resource_path: false, + requires_resource_path: true, }, ScopeOption { value: "jobs:write".to_string(), label: "Write".to_string(), requires_resource_path: false, }, + ScopeOption { + value: "jobs:cancel".to_string(), + label: "Cancel".to_string(), + requires_resource_path: true, + }, ScopeOption { value: "jobs:run:scripts".to_string(), label: "Run scripts".to_string(), @@ -210,6 +215,11 @@ lazy_static! { label: "Run flows".to_string(), requires_resource_path: true, }, + ScopeOption { + value: "jobs:run:dependencies".to_string(), + label: "Generate dependency locks".to_string(), + requires_resource_path: true, + }, ], }]; diff --git a/backend/windmill-api/src/users.rs b/backend/windmill-api/src/users.rs index a7f9aeda89..5b26cf57b8 100644 --- a/backend/windmill-api/src/users.rs +++ b/backend/windmill-api/src/users.rs @@ -198,6 +198,7 @@ async fn set_password( Extension(db): Extension, Extension(argon2): Extension>>, authed: ApiAuthed, + Tokened { token }: Tokened, OptJobAuthed { job_id, .. }: OptJobAuthed, Json(ep): Json, ) -> Result { @@ -206,7 +207,15 @@ async fn set_password( // (GHSA-hfh4-cx4h-3fcr). forbid_elevated_job_token(&db, &authed.email, job_id).await?; let email = authed.email.clone(); - crate::users_oss::set_password(db, argon2, authed, &email, ep).await + let res = crate::users_oss::set_password(db.clone(), argon2, authed, &email, ep).await?; + sqlx::query!( + "DELETE FROM token WHERE email = $1 AND label = 'session' AND token_hash != $2", + &email, + windmill_common::auth::hash_token(&token) + ) + .execute(&db) + .await?; + Ok(res) } async fn set_password_of_user( @@ -219,7 +228,14 @@ async fn set_password_of_user( ) -> Result { require_super_admin(&db, &authed).await?; forbid_superadmin_job_token(&db, &authed.email, job_id).await?; - crate::users_oss::set_password(db, argon2, authed, &email, ep).await + let res = crate::users_oss::set_password(db.clone(), argon2, authed, &email, ep).await?; + sqlx::query!( + "DELETE FROM token WHERE email = $1 AND label = 'session'", + &email + ) + .execute(&db) + .await?; + Ok(res) } #[derive(Deserialize)] @@ -665,8 +681,8 @@ async fn update_username_in_workpsace<'c>( // ---- flows ---- sqlx::query!( r#"INSERT INTO flow - (workspace_id, path, summary, description, archived, extra_perms, dependency_job, tag, ws_error_handler_muted, dedicated_worker, timeout, visible_to_runner_only, on_behalf_of, on_behalf_of_email, concurrency_key, versions, value, schema, edited_by, edited_at) - SELECT workspace_id, REGEXP_REPLACE(path,'u/' || $2 || '/(.*)','u/' || $1 || '/\1'), summary, description, archived, extra_perms, dependency_job, tag, ws_error_handler_muted, dedicated_worker, timeout, visible_to_runner_only, on_behalf_of, on_behalf_of_email, concurrency_key, versions, value, schema, edited_by, edited_at + (workspace_id, path, summary, description, archived, extra_perms, dependency_job, tag, ws_error_handler_muted, dedicated_worker, timeout, visible_to_runner_only, on_behalf_of, on_behalf_of_email, concurrency_key, versions, value, schema, edited_by, edited_at, job_token_scopes) + SELECT workspace_id, REGEXP_REPLACE(path,'u/' || $2 || '/(.*)','u/' || $1 || '/\1'), summary, description, archived, extra_perms, dependency_job, tag, ws_error_handler_muted, dedicated_worker, timeout, visible_to_runner_only, on_behalf_of, on_behalf_of_email, concurrency_key, versions, value, schema, edited_by, edited_at, job_token_scopes FROM flow WHERE path LIKE ('u/' || $2 || '/%') AND workspace_id = $3"#, new_username, diff --git a/backend/windmill-api/src/workspaces_export.rs b/backend/windmill-api/src/workspaces_export.rs index ea79aaa572..3c3e965248 100644 --- a/backend/windmill-api/src/workspaces_export.rs +++ b/backend/windmill-api/src/workspaces_export.rs @@ -119,6 +119,8 @@ struct ScriptMetadata { pub debouncing_settings: DebouncingSettings, #[serde(skip_serializing_if = "Option::is_none")] pub labels: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub job_token_scopes: Option>, #[serde(skip_serializing_if = "is_empty_extra_perms")] pub extra_perms: serde_json::Value, } @@ -906,6 +908,7 @@ pub(crate) async fn tarball_workspace( .then_some(true), modules: script.modules, labels: script.labels, + job_token_scopes: script.job_token_scopes, // Same opt-in contract as flow/app: the tarball only surfaces // ACLs when `?preserve_extra_perms=true`. Passing `Null` lets the // `is_empty_extra_perms` skip-serializer drop the field entirely. @@ -966,7 +969,7 @@ pub(crate) async fn tarball_workspace( { let flows = sqlx::query_as::<_, Flow>( - "SELECT flow.workspace_id, flow.path, flow.summary, flow.description, flow.archived, flow.extra_perms, flow.dedicated_worker, flow.tag, flow.ws_error_handler_muted, flow.timeout, flow.visible_to_runner_only, flow.on_behalf_of, flow.labels, flow_version.schema, flow_version.value, flow_version.created_at as edited_at, flow_version.created_by as edited_by + "SELECT flow.workspace_id, flow.path, flow.summary, flow.description, flow.archived, flow.extra_perms, flow.dedicated_worker, flow.tag, flow.ws_error_handler_muted, flow.timeout, flow.visible_to_runner_only, flow.on_behalf_of, flow.labels, flow.job_token_scopes, flow_version.schema, flow_version.value, flow_version.created_at as edited_at, flow_version.created_by as edited_by FROM flow LEFT JOIN flow_version ON flow_version.id = flow.versions[array_upper(flow.versions, 1)] WHERE flow.workspace_id = $1 AND flow.archived = false", diff --git a/backend/windmill-audit/Cargo.toml b/backend/windmill-audit/Cargo.toml index 6e4fc3fd12..8c26615664 100644 --- a/backend/windmill-audit/Cargo.toml +++ b/backend/windmill-audit/Cargo.toml @@ -20,4 +20,7 @@ chrono.workspace = true serde_json.workspace = true tracing.workspace = true windmill-common = { workspace = true, default-features = false } -lazy_static.workspace = true \ No newline at end of file +lazy_static.workspace = true + +[dev-dependencies] +tracing-subscriber.workspace = true \ No newline at end of file diff --git a/backend/windmill-audit/src/audit_oss.rs b/backend/windmill-audit/src/audit_oss.rs index da36ada468..02b075af9d 100644 --- a/backend/windmill-audit/src/audit_oss.rs +++ b/backend/windmill-audit/src/audit_oss.rs @@ -88,6 +88,21 @@ pub async fn audit_log<'c, E: sqlx::Executor<'c, Database = Postgres>>( Ok(()) } +/// AUTHORIZATION: records entries, checks nothing. The caller must have authorized `author` +/// for the operation on every resource in `w_id` before calling. +#[cfg(not(feature = "private"))] +pub async fn audit_log_many<'c, E: sqlx::Executor<'c, Database = Postgres>>( + _db: E, + _author: &impl AuditAuthorable, + _operation: &str, + _action_kind: ActionKind, + _w_id: &str, + _resources: &[String], +) -> Result<()> { + // Implementation is not open source as Audit logs is a Windmill Enterprise Edition feature + Ok(()) +} + #[cfg(not(feature = "private"))] pub async fn list_audit( _tx: Transaction<'_, Postgres>, diff --git a/backend/windmill-audit/src/instance_config.rs b/backend/windmill-audit/src/instance_config.rs new file mode 100644 index 0000000000..977e0fe6c6 --- /dev/null +++ b/backend/windmill-audit/src/instance_config.rs @@ -0,0 +1,148 @@ +//! Audit entries for writes to instance settings and worker group configs, whichever way +//! they arrive: an API request, or a declarative sync that no user request stands behind. + +use std::collections::HashMap; + +use windmill_common::{ + audit::AuditAuthor, error::Result, instance_config::ConfigWriteObserver, + users::SUPERADMIN_SYNC_EMAIL, DB, +}; + +use crate::{ + audit_oss::{audit_log, AuditAuthorable}, + ActionKind, +}; + +/// The declarative writer behind an entry, recorded as its `source` parameter. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum DeclarativeSource { + /// `windmill sync-config ` + SyncConfig, + /// The Kubernetes operator's ConfigMap sync. + Operator, +} + +impl DeclarativeSource { + pub fn as_str(&self) -> &'static str { + match self { + DeclarativeSource::SyncConfig => "sync-config", + DeclarativeSource::Operator => "operator", + } + } +} + +/// Who a declarative sync's writes are attributed to. It runs on direct database +/// credentials, so there is no user to name. +pub fn declarative_sync_author() -> AuditAuthor { + AuditAuthor { + username: "instance_config_sync".to_string(), + email: SUPERADMIN_SYNC_EMAIL.to_string(), + username_override: None, + token_prefix: None, + } +} + +fn parameters<'a>( + value: Option<&'a str>, + source: Option, +) -> Option> { + let parameters: HashMap<&str, &str> = value + .map(|v| ("value", v)) + .into_iter() + .chain(source.map(|s| ("source", s.as_str()))) + .collect(); + (!parameters.is_empty()).then_some(parameters) +} + +/// A setting is recorded by name, with its value only when that is a boolean or a number. +/// Any other shape can carry a credential (a registry URL, a webhook, a header map), and no +/// list of secret-bearing settings stays complete as settings are added. +/// +/// AUTHORIZATION: records a write, checks nothing. The caller must have established that +/// `author` may change instance settings before performing the write this records. +pub async fn audit_setting_write( + db: &DB, + author: &impl AuditAuthorable, + key: &str, + value: Option<&serde_json::Value>, + source: Option, +) -> Result<()> { + let deleted = match value { + None | Some(serde_json::Value::Null) => true, + Some(serde_json::Value::String(s)) => s.trim().is_empty(), + Some(_) => false, + }; + let recorded = match value { + Some(v @ (serde_json::Value::Bool(_) | serde_json::Value::Number(_))) => { + Some(v.to_string()) + } + _ => None, + }; + audit_log( + db, + author, + if deleted { + "global_settings.delete" + } else { + "global_settings.update" + }, + if deleted { + ActionKind::Delete + } else { + ActionKind::Update + }, + "global", + Some(key), + parameters(recorded.as_deref(), source), + ) + .await +} + +/// A worker group config is recorded by name only: its env vars and init script are free +/// text that can carry a credential. +/// +/// AUTHORIZATION: records a write, checks nothing. The caller must have established that +/// `author` may change worker group configs before performing the write this records. +pub async fn audit_worker_config_write( + db: &DB, + author: &impl AuditAuthorable, + group: &str, + deleted: bool, + source: Option, +) -> Result<()> { + let (operation, action_kind) = if deleted { + ("worker_config.delete", ActionKind::Delete) + } else { + ("worker_config.update", ActionKind::Update) + }; + audit_log( + db, + author, + operation, + action_kind, + "global", + Some(&format!("worker__{group}")), + parameters(None, source), + ) + .await +} + +/// Audits each write of an instance config apply as `author`'s. +/// +/// AUTHORIZATION: as [`audit_setting_write`], for every write it is told of. +pub struct InstanceConfigAuditor<'a, A> { + pub db: &'a DB, + pub author: &'a A, + /// `None` for a write made through the API. + pub source: Option, +} + +impl ConfigWriteObserver for InstanceConfigAuditor<'_, A> { + async fn setting_written(&self, key: &str, value: Option<&serde_json::Value>) -> Result<()> { + audit_setting_write(self.db, self.author, key, value, self.source).await + } + + async fn worker_config_written(&self, group: &str, deleted: bool) -> Result<()> { + audit_worker_config_write(self.db, self.author, group, deleted, self.source).await + } +} diff --git a/backend/windmill-audit/src/lib.rs b/backend/windmill-audit/src/lib.rs index f1443fdb18..e9c71f1336 100644 --- a/backend/windmill-audit/src/lib.rs +++ b/backend/windmill-audit/src/lib.rs @@ -4,6 +4,7 @@ use sqlx::FromRow; #[cfg(feature = "private")] pub mod audit_ee; pub mod audit_oss; +pub mod instance_config; #[derive(sqlx::Type, Serialize, Deserialize, Debug)] #[sqlx(type_name = "ACTION_KIND", rename_all = "lowercase")] diff --git a/backend/windmill-common/Cargo.toml b/backend/windmill-common/Cargo.toml index 96fa7ceb63..e447baf563 100644 --- a/backend/windmill-common/Cargo.toml +++ b/backend/windmill-common/Cargo.toml @@ -8,7 +8,7 @@ edition.workspace = true default = [] enterprise = ["dep:aws-config"] instance_config_schema = ["dep:schemars"] -local_reports = ["dep:rsa", "dep:aes-gcm"] +local_reports = ["dep:rsa"] private = ["dep:aws-sdk-rds", "dep:aws-sdk-secretsmanager", "dep:aws-config"] jemalloc = ["dep:tikv-jemalloc-ctl"] tantivy = [] @@ -22,7 +22,7 @@ smtp = ["dep:mail-send", "dep:rustls", "dep:tokio-rustls", "dep:rustls-native-ce scoped_cache = [] cloud = [] dev_override = [] -openidconnect = ["dep:openidconnect"] +openidconnect = ["dep:openidconnect", "dep:rsa", "dep:p256"] python = ["dep:windmill-parser-py"] [lib] @@ -33,6 +33,7 @@ path = "src/lib.rs" tar.workspace = true hmac.workspace = true sha2.workspace = true +ryu-js.workspace = true sha1.workspace = true thiserror.workspace = true anyhow.workspace = true @@ -115,9 +116,11 @@ pep440_rs.workspace = true systemstat.workspace = true size.workspace = true rsa = { workspace = true, optional = true } +p256 = { workspace = true, optional = true } spki = { workspace = true } pkcs1 = { workspace = true } -aes-gcm = { workspace = true, optional = true } +aes-gcm.workspace = true +hkdf.workspace = true semver.workspace = true croner.workspace = true diff --git a/backend/windmill-common/src/apps.rs b/backend/windmill-common/src/apps.rs index 36b2d9b704..1686c20d2b 100644 --- a/backend/windmill-common/src/apps.rs +++ b/backend/windmill-common/src/apps.rs @@ -18,6 +18,59 @@ lazy_static::lazy_static! { pub static ref APP_WORKSPACED_ROUTE: AtomicBool = AtomicBool::new(false); } +/// Whether the app value carries an `inlineScript` anywhere. +/// +/// Deliberately not built on [`traverse_app_inline_scripts`], which only reports a script whose +/// `language` parses and stops descending as soon as it sees an `inlineScript` key. That is right +/// for locking (nothing to lock without a language) and wrong for an authorization check, where +/// the author picks the fields: this refuses the key itself, whatever it contains. +pub fn app_value_has_inline_script(value: &Value) -> bool { + match value { + Value::Object(object) => { + object.contains_key("inlineScript") || object.values().any(app_value_has_inline_script) + } + Value::Array(array) => array.iter().any(app_value_has_inline_script), + _ => false, + } +} + +/// Every workspace runnable the app value points a component at, as `(is_flow, path)`. +/// +/// This is what the deployed bundle actually asks `execute_component` to run: it resolves a +/// `runnable_id` against the stored `runnables` and sends `{runType}/{path}`. The policy's +/// triggerables are a separate surface, so both have to be authorized. Anything not plain text +/// is refused, never skipped: the editor compares `type` loosely (`["path"] == "path"`) and +/// interpolates `path` (`["u/x"]` → `u/x`), so a skipped entry still runs. +pub fn app_value_runnable_paths(value: &Value) -> error::Result> { + let bad = |msg: String| error::Error::BadRequest(msg); + let runnables = match value.get("runnables") { + None | Some(Value::Null) => return Ok(vec![]), + Some(Value::Object(runnables)) => runnables, + Some(_) => return Err(bad("App runnables must be an object".to_string())), + }; + let mut out = Vec::new(); + for (id, runnable) in runnables { + let Value::Object(runnable) = runnable else { + return Err(bad(format!("Runnable {id} must be an object"))); + }; + match runnable.get("type") { + Some(Value::String(t)) if t == "runnableByPath" || t == "path" => {} + None | Some(Value::String(_)) => continue, + Some(_) => return Err(bad(format!("Runnable {id} has a non-text type"))), + } + let Some(Value::String(path)) = runnable.get("path") else { + return Err(bad(format!("Runnable {id} must name its path as text"))); + }; + let is_flow = match runnable.get("runType") { + Some(Value::String(t)) if t == "flow" => true, + Some(Value::String(t)) if t == "script" || t == "hubscript" => false, + _ => return Err(bad(format!("Unsupported runType for runnable {path}"))), + }; + out.push((is_flow, path.clone())); + } + Ok(out) +} + /// Traverse FlowValue while invoking provided by caller callback on leafs // #[async_recursion::async_recursion(?Send)] pub fn traverse_app_inline_scripts< diff --git a/backend/windmill-common/src/auth.rs b/backend/windmill-common/src/auth.rs index 86b7f173ac..17310bdbbc 100644 --- a/backend/windmill-common/src/auth.rs +++ b/backend/windmill-common/src/auth.rs @@ -362,6 +362,10 @@ pub struct JobPerms { pub groups: Vec, pub folders: Vec, pub end_user_email: Option, + /// The job's effective scopes, stored on its `job_perms` row at push and minted into its + /// token. + #[serde(default)] + pub job_token_scopes: Option>, } impl From for Authed { @@ -654,7 +658,9 @@ pub async fn get_job_perms<'a, E: sqlx::PgExecutor<'a>>( ) -> sqlx::Result> { sqlx::query_as!( JobPerms, - "SELECT email, username, is_admin, is_operator, groups, folders, end_user_email FROM job_perms WHERE job_id = $1 AND workspace_id = $2", + "SELECT email, username, is_admin, is_operator, groups, folders, end_user_email, + job_token_scopes + FROM job_perms WHERE job_id = $1 AND workspace_id = $2", job_id, w_id ) @@ -745,17 +751,29 @@ pub async fn create_token_for_owner( } else { get_job_perms(db, job_id, w_id).await }; - let job_authed = match job_perms { - Ok(Some(jp)) => jp.into(), - _ => { + let (job_authed, job_token_scopes) = match job_perms { + Ok(Some(mut jp)) => { + let scopes = jp.job_token_scopes.take(); + (jp.into(), scopes) + } + // A failed read must not mint as if the job had no row: that would drop its restriction. + Err(e) => { + return Err(Error::internal_err(format!( + "Could not read permissions for job {job_id}: {e:#}" + ))) + } + // Push writes a job's `job_perms` row and its scopes in one statement, so a job with no + // row was never restricted. + Ok(None) => { tracing::warn!("Could not get permissions for job {job_id} from job_perms table, getting permissions directly..."); - fetch_authed_from_permissioned_as(owner, email, w_id, db) + let authed = fetch_authed_from_permissioned_as(owner, email, w_id, db) .await .map_err(|e| { Error::internal_err(format!( "Could not get permissions directly for job {job_id}: {e:#}" )) - })? + })?; + (authed, None) } }; @@ -766,7 +784,7 @@ pub async fn create_token_for_owner( Some(*job_id), Some(label.to_string()), audit_span, - None, + crate::scopes::job_token_jwt_scopes(job_token_scopes), ) .await } diff --git a/backend/windmill-common/src/client.rs b/backend/windmill-common/src/client.rs index 48f3e0205b..985d3bbbfc 100644 --- a/backend/windmill-common/src/client.rs +++ b/backend/windmill-common/src/client.rs @@ -307,12 +307,13 @@ impl AuthedClient { workspace_id: &str, file_key: &str, storage: Option, + max_bytes: Option, ) -> anyhow::Result { let mut query = vec![("file_key", file_key.to_string())]; if let Some(storage) = storage { query.push(("storage", storage)); } - let response = self + let mut response = self .force_client .as_ref() .unwrap_or(&HTTP_CLIENT) @@ -331,6 +332,39 @@ impl AuthedClient { .context("Failed to send download_s3_file request") .map_err(|e| anyhow::anyhow!(e.to_string()))?; + if let Some(max_bytes) = max_bytes { + let size_error = || { + anyhow::anyhow!( + "S3 file '{file_key}' exceeds maximum supported download size of {max_bytes} bytes" + ) + }; + if response + .content_length() + .is_some_and(|len| len > max_bytes as u64) + { + return Err(size_error()); + } + + let status = response.status(); + let mut bytes = Vec::new(); + // Content-Length may be absent (including for decoded compressed responses). + while let Some(chunk) = response + .chunk() + .await + .context("Failed to read response bytes")? + { + if chunk.len() > max_bytes - bytes.len() { + return Err(size_error()); + } + bytes.extend_from_slice(&chunk); + } + return if status == reqwest::StatusCode::OK { + Ok(bytes.into()) + } else { + Err(anyhow::anyhow!(String::from_utf8_lossy(&bytes).into_owned())) + }; + } + match response.status().as_u16() { 200u16 => Ok(response .bytes() @@ -341,6 +375,78 @@ impl AuthedClient { } } +#[cfg(test)] +mod tests { + use super::*; + use std::time::Duration; + use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader}; + + async fn download_response( + response: &'static str, + max_bytes: Option, + ) -> anyhow::Result { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let client = AuthedClient::new( + format!("http://{}", listener.local_addr().unwrap()), + "test_ws".to_string(), + "test_token".to_string(), + Some(reqwest::Client::builder().no_proxy().build().unwrap()), + ); + let server = tokio::spawn(async move { + let (socket, _) = listener.accept().await.unwrap(); + let mut socket = BufReader::new(socket); + loop { + let mut line = String::new(); + assert!(socket.read_line(&mut line).await.unwrap() > 0); + if line == "\r\n" { + break; + } + } + socket + .get_mut() + .write_all(response.as_bytes()) + .await + .unwrap(); + // Keep incomplete bodies open so rejection must happen before EOF. + std::future::pending::<()>().await; + }); + let result = tokio::time::timeout( + Duration::from_secs(5), + client.download_s3_file("test_ws", "image.png", None, max_bytes), + ) + .await; + server.abort(); + result.expect("download waited for the oversized body to finish") + } + + #[tokio::test] + async fn download_s3_file_rejects_oversized_body_before_eof() { + for response in [ + "HTTP/1.1 200 OK\r\nContent-Length: 5\r\n\r\n", + "HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n4\r\nabcd\r\n1\r\ne\r\n", + "HTTP/1.1 500 Internal Server Error\r\nTransfer-Encoding: chunked\r\n\r\n5\r\nerror\r\n", + ] { + let error = download_response(response, Some(4)).await.unwrap_err(); + assert!(error.to_string().contains( + "S3 file 'image.png' exceeds maximum supported download size of 4 bytes" + )); + } + } + + #[tokio::test] + async fn download_s3_file_accepts_body_at_limit() { + for response in [ + "HTTP/1.1 200 OK\r\nContent-Length: 4\r\n\r\nabcd", + "HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n4\r\nabcd\r\n0\r\n\r\n", + ] { + assert_eq!( + download_response(response, Some(4)).await.unwrap(), + &b"abcd"[..] + ); + } + } +} + #[inline] fn query_from_json_path(json_path: Option) -> Vec<(&'static str, String)> { json_path diff --git a/backend/windmill-common/src/datatable_roles.rs b/backend/windmill-common/src/datatable_roles.rs index f355e733cb..522327040d 100644 --- a/backend/windmill-common/src/datatable_roles.rs +++ b/backend/windmill-common/src/datatable_roles.rs @@ -72,6 +72,11 @@ impl DatatableRoleCluster { /// created, renamed or dropped. pub const ADMIN_DATATABLE_ROLE: &str = "admin"; +/// Stands in a data table's `default_role` for "none": a connection that names no role then holds +/// its viewer's roles only, and is refused without any. `$` is outside both the role-name charset +/// and the catalog ids, and a server predating it refuses it as a role the data table lacks. +pub const NO_DEFAULT_DATATABLE_ROLE: &str = "$none"; + /// The login the admin connection uses, and the role every created role is granted to — that /// membership is what later lets it `ALTER ... OWNER TO` a role and drop it. pub const CUSTOM_INSTANCE_USER: &str = "custom_instance_user"; @@ -121,8 +126,13 @@ fn is_reserved_role_name(name: &str) -> bool { || lower.starts_with("pg_") || lower.starts_with("windmill_") || lower.starts_with("custom_instance_") + || lower.starts_with(ROLE_SET_LOGIN_PREFIX) } +/// Prefix of the logins Windmill creates for a connection made as several roles at once. Reserved, +/// so no data table role can be created under, renamed to or taken over at such a name. +pub const ROLE_SET_LOGIN_PREFIX: &str = "wm_set_"; + /// The charset is what makes every downstream interpolation safe: the name reaches Postgres as a /// quoted identifier, a `-- role ` annotation, and a `?role=` query parameter. pub fn validate_role_name(name: &str) -> Result<()> { @@ -331,6 +341,12 @@ pub async fn converge_connect_grants_with( /// `CREATE ROLE LOGIN PASSWORD ...; GRANT TO custom_instance_user` on `cluster`. No /// privileges beyond that — an admin grants them through SQL or the ACL editor. /// +/// A role of that name that already exists is refused with a 409, unless `take_over`: then its +/// password is replaced and it is managed from here on like one created here. A role holding +/// cluster-wide privileges, or inheriting another role's, is refused either way, since every tenant +/// would get them. Returns whether an existing role was taken over: undoing a failed take-over +/// must not drop a role that was there before. +/// /// On Windmill's own cluster the DDL runs on `tx`, so it commits with the catalog row. The external /// cluster is another server: the role is created there before `tx` commits, and callers MUST drop /// it again ([`drop_datatable_role`]) if `tx` then fails to commit. @@ -343,8 +359,10 @@ pub async fn create_datatable_role( cluster: DatatableRoleCluster, name: &str, password: &str, -) -> Result<()> { - crate::datatable_roles_oss::create_datatable_role(db, tx, cluster, name, password).await + take_over: bool, +) -> Result { + crate::datatable_roles_oss::create_datatable_role(db, tx, cluster, name, password, take_over) + .await } /// Authorization: alters a cluster-wide Postgres login. Callers MUST restrict this to superadmin @@ -406,6 +424,13 @@ pub async fn drop_datatable_role( crate::datatable_roles_oss::drop_datatable_role(db, tx, cluster, name).await } +/// Drop the logins made for a connection as several roles at once that nobody has connected with +/// for a month, on every cluster. They are made again on demand, so this only bounds how many +/// exist. Safe to call from any server at any time: it serializes on the role catalog lock. +pub async fn sweep_unused_role_set_logins(db: &DB) -> Result<()> { + crate::datatable_roles_oss::sweep_unused_role_set_logins(db).await +} + #[cfg(test)] mod tests { use super::*; diff --git a/backend/windmill-common/src/datatable_roles_oss.rs b/backend/windmill-common/src/datatable_roles_oss.rs index a97a936827..739f08d60b 100644 --- a/backend/windmill-common/src/datatable_roles_oss.rs +++ b/backend/windmill-common/src/datatable_roles_oss.rs @@ -29,7 +29,8 @@ pub(crate) use crate::datatable_roles_ee::{ ensure_instance_db_grant_options_unchecked, forget_datatable_role_everywhere, insert_role_catalog_entry, read_role_catalog, read_role_catalog_tx, registered_instance_databases, rename_datatable_role, resolve_datatable_role_connection, - role_cluster, set_datatable_role_login, update_role_catalog_entry, + role_cluster, set_datatable_role_login, sweep_unused_role_set_logins, + update_role_catalog_entry, }; #[cfg(not(all(feature = "private", feature = "enterprise")))] @@ -125,7 +126,8 @@ mod ce { _cluster: DatatableRoleCluster, _name: &str, _password: &str, - ) -> Result<()> { + _take_over: bool, + ) -> Result { Err(unavailable()) } @@ -234,6 +236,11 @@ mod ce { } } + /// No roles, so no set login to sweep. + pub(crate) async fn sweep_unused_role_set_logins(_db: &DB) -> Result<()> { + Ok(()) + } + pub(crate) async fn forget_datatable_role_everywhere( _tx: &mut Tx<'_>, _role_id: &str, diff --git a/backend/windmill-common/src/external_instance_pg.rs b/backend/windmill-common/src/external_instance_pg.rs index c0b6203e40..72920a1874 100644 --- a/backend/windmill-common/src/external_instance_pg.rs +++ b/backend/windmill-common/src/external_instance_pg.rs @@ -198,7 +198,8 @@ pub async fn external_instance_database_usages<'c>( /// would be a login nothing can drop any more. Allowed on every edition, so a /// downgraded instance can still clear a setting it no longer uses. async fn ensure_external_instance_pg_removable(conn: &mut sqlx::PgConnection) -> Result<()> { - ensure_external_instance_pg_unused(conn, &format!("removing {EXTERNAL_INSTANCE_PG_SETTING}")).await + ensure_external_instance_pg_unused(conn, &format!("removing {EXTERNAL_INSTANCE_PG_SETTING}")) + .await } /// Refuse while Windmill has databases or data table roles on the cluster, or a workspace points @@ -394,16 +395,21 @@ pub async fn write_external_instance_pg_setting( } /// [`write_external_instance_pg_setting`] for a settings diff: writes the key if the diff touches -/// it, and takes it out of the diff so the generic apply does not write it again. +/// it, and takes it out of the diff so the generic apply does not write it again. `observer` is +/// told of the write here, since the generic apply no longer sees the key. /// /// Authorization: checks nothing. Callers MUST be superadmin, or the declarative instance config /// sync, which applies what the operator deployed. pub async fn write_external_instance_pg_from_diff( db: &DB, diff: &mut crate::instance_config::SettingsDiff, + observer: &impl crate::instance_config::ConfigWriteObserver, ) -> Result<()> { if let Some(value) = diff.upserts.remove(EXTERNAL_INSTANCE_PG_SETTING) { write_external_instance_pg_setting(db, Some(&value)).await?; + observer + .setting_written(EXTERNAL_INSTANCE_PG_SETTING, Some(&value)) + .await?; } if let Some(i) = diff .deletes @@ -412,6 +418,9 @@ pub async fn write_external_instance_pg_from_diff( { diff.deletes.remove(i); write_external_instance_pg_setting(db, None).await?; + observer + .setting_written(EXTERNAL_INSTANCE_PG_SETTING, None) + .await?; } Ok(()) } diff --git a/backend/windmill-common/src/flow_conversations.rs b/backend/windmill-common/src/flow_conversations.rs index 5ee99d0681..bde4d5ccce 100644 --- a/backend/windmill-common/src/flow_conversations.rs +++ b/backend/windmill-common/src/flow_conversations.rs @@ -278,6 +278,8 @@ pub async fn add_message_to_conversation_tx( tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, conversation_id: Uuid, job_id: Option, + // The flow run the user message started; retention deletes the turn's rows by it. + turn_job_id: Uuid, content: &str, message_type: MessageType, step_name: Option<&str>, @@ -303,12 +305,13 @@ pub async fn add_message_to_conversation_tx( // Insert the message sqlx::query!( - "INSERT INTO flow_conversation_message (conversation_id, message_type, content, job_id, step_name, success, tool_arguments, tool_result, reasoning, attachments) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)", + "INSERT INTO flow_conversation_message (conversation_id, message_type, content, job_id, turn_job_id, step_name, success, tool_arguments, tool_result, reasoning, attachments) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11)", conversation_id, message_type as MessageType, content, job_id, + turn_job_id, step_name, success, extras.and_then(|e| e.tool_arguments.as_deref()), diff --git a/backend/windmill-common/src/flows.rs b/backend/windmill-common/src/flows.rs index b392dda89a..222694aeac 100644 --- a/backend/windmill-common/src/flows.rs +++ b/backend/windmill-common/src/flows.rs @@ -357,6 +357,8 @@ fn check_module_value_is_composition_only( } } } + // Runs no code: it sends its inputs to the decision model. + FlowModuleValue::AIDecision { tag, .. } => push_tag(tag), } Ok(()) } diff --git a/backend/windmill-common/src/global_settings.rs b/backend/windmill-common/src/global_settings.rs index 9137dc364e..2bc303f5c2 100644 --- a/backend/windmill-common/src/global_settings.rs +++ b/backend/windmill-common/src/global_settings.rs @@ -9,6 +9,7 @@ pub const DEFAULT_TAGS_WORKSPACES_SETTING: &str = "default_tags_workspaces"; pub const FORK_WORKSPACE_TAG_APPEND_FORK_SUFFIX_SETTING: &str = "fork_workspace_tag_append_fork_suffix"; pub const PREVIEW_TAGS_OVERRIDE_SETTING: &str = "preview_tags_override"; +pub const DEPENDENCY_JOB_TAG_SETTING: &str = "dependency_job_tag"; pub const BASE_URL_SETTING: &str = "base_url"; pub const WS_BASE_URL_SETTING: &str = "ws_base_url"; pub const OAUTH_SETTING: &str = "oauths"; @@ -157,6 +158,7 @@ pub const SSO_GROUPS_CLAIM_SETTING: &str = "sso_groups_claim"; pub const HUB_BASE_URL_SETTING: &str = "hub_base_url"; pub const HUB_ACCESSIBLE_URL_SETTING: &str = "hub_accessible_url"; pub const DISABLE_HUB_SETTING: &str = "disable_hub"; +pub const SYNC_HUB_RESOURCE_TYPES_DAILY_SETTING: &str = "sync_hub_resource_types_daily"; pub const CRITICAL_ERROR_CHANNELS_SETTING: &str = "critical_error_channels"; pub const CRITICAL_ALERT_MUTE_UI_SETTING: &str = "critical_alert_mute_ui"; pub const CRITICAL_ALERTS_ON_DB_OVERSIZE_SETTING: &str = "critical_alerts_on_db_oversize"; @@ -180,6 +182,12 @@ pub const GITHUB_ENTERPRISE_APP_SETTING: &str = "github_enterprise_app"; /// `base_url` when unset; set it when the browser-facing URL is not reachable /// from GitHub and a separate ingress fronts the API for inbound webhooks. pub const GITHUB_APP_WEBHOOK_BASE_URL_SETTING: &str = "github_app_webhook_base_url"; +/// Base URL the UI shows for the endpoints external clients call (webhooks, HTTP +/// routes, push trigger endpoints, the CLI), instead of the origin the UI is browsed on. +/// For deployments that serve the API on a separate domain or behind a gateway. +/// EE only. Readable by any authenticated user, like [`INSTANCE_BANNER_SETTING`]: +/// every user is shown URLs built from it. +pub const API_BASE_URL_SETTING: &str = "api_base_url"; /// Instance-wide announcement rendered above every page of the app (maintenance /// windows, incidents). Readable by any authenticated user, unlike most settings: /// the banner exists to be shown to everyone, so it must never hold anything the @@ -312,12 +320,17 @@ pub fn validate_accent_color(value: &serde_json::Value) -> Result<(), String> { pub struct InstanceUi { pub instance_banner: Option, pub accent_color: Option, + pub api_base_url: Option, } pub async fn get_instance_ui(db: &Pool) -> error::Result { let rows = sqlx::query!( "SELECT name, value FROM global_settings WHERE name = ANY($1)", - &[INSTANCE_BANNER_SETTING, ACCENT_COLOR_SETTING] as &[&str] + &[ + INSTANCE_BANNER_SETTING, + ACCENT_COLOR_SETTING, + API_BASE_URL_SETTING + ] as &[&str] ) .fetch_all(db) .await?; @@ -326,17 +339,22 @@ pub async fn get_instance_ui(db: &Pool) -> error::Result { match row.name.as_str() { INSTANCE_BANNER_SETTING => ui.instance_banner = Some(row.value), ACCENT_COLOR_SETTING => ui.accent_color = Some(row.value), + API_BASE_URL_SETTING => ui.api_base_url = Some(row.value), _ => {} } } Ok(ui) } -/// Validate a [`GITHUB_APP_WEBHOOK_BASE_URL_SETTING`] value. +const SHELL_ACTIVE_CHARS: [char; 16] = [ + '$', '`', '\'', '"', '\\', ';', '&', '|', '<', '>', '(', ')', '{', '}', '*', '!', +]; + +/// Validate a [`GITHUB_APP_WEBHOOK_BASE_URL_SETTING`] or [`API_BASE_URL_SETTING`] value. /// -/// The receiver path is appended to it verbatim, so anything that doesn't -/// concatenate into a URL GitHub can POST to must be rejected at write time -/// rather than silently producing an unreachable hook: a wrong scheme +/// A path is appended to it verbatim, so anything that doesn't concatenate into a +/// URL an external caller can reach must be rejected at write time rather than +/// silently producing an unreachable hook or endpoint: a wrong scheme /// (`httpss://`), a missing host, embedded whitespace, or a query/fragment /// (appending a path after `?`/`#` keeps it inside the query/fragment). /// @@ -352,7 +370,7 @@ pub fn validate_webhook_base_url(value: &str) -> Result<(), String> { url::Url::parse(value).map_err(|e| format!("must be an absolute http(s) URL: {e}"))?; if !url.username().is_empty() || url.password().is_some() { return Err( - "must not embed a username or password: the receiver URL is stored in workspace settings, where it is readable by workspace admins".to_string(), + "must not embed a username or password: the URL is readable by users who are not instance admins".to_string(), ); } if !matches!(url.scheme(), "http" | "https") { @@ -365,13 +383,21 @@ pub fn validate_webhook_base_url(value: &str) -> Result<(), String> { } if url.query().is_some() || url.fragment().is_some() { return Err( - "must not include a query string or fragment, since the webhook path is appended to it" + "must not include a query string or fragment, since a path is appended to it" .to_string(), ); } if value.chars().any(char::is_whitespace) { return Err("must not contain whitespace".to_string()); } + // The UI pastes the URL into shell commands users copy, where double quotes + // do not stop `$(...)` or backticks from running. + if value.contains(SHELL_ACTIVE_CHARS) { + return Err( + "must not contain shell metacharacters, since it is shown inside commands users copy" + .to_string(), + ); + } // Matches the sibling `base_url` / `hub_base_url` convention. The receiver // builder trims it anyway, so this is about keeping the stored value canonical // rather than about reachability. @@ -430,8 +456,9 @@ pub const CANCEL_STRANDED_JOBS_AFTER_DAYS_SETTING: &str = "cancel_stranded_jobs_ /// server keeps to itself, add it here. pub const AGENT_WORKER_BLOCKED_SETTINGS: &[&str] = &[ // Instance identity / auth secrets — disclosure enables privilege escalation - // or impersonation. + // or impersonation. `rsa_keys` signs job OIDC tokens; only the server mints them. JWT_SECRET_SETTING, + "rsa_keys", OAUTH_SETTING, SMTP_SETTING, SCIM_TOKEN_SETTING, @@ -1099,6 +1126,9 @@ mod tests { "https://hooks.example.com#", "https://user:password@hooks.example.com", "https://user@hooks.example.com", + "https://hooks.example.com/$(uname)", + "https://hooks.example.com/`uname`", + "https://hooks.example.com/a;b", ]; for v in accept { assert!( @@ -1143,6 +1173,7 @@ mod tests { // secrets. They must never be served by the agent-worker endpoint. for key in [ JWT_SECRET_SETTING, + "rsa_keys", OAUTH_SETTING, SMTP_SETTING, SCIM_TOKEN_SETTING, diff --git a/backend/windmill-common/src/hub_resource_types.rs b/backend/windmill-common/src/hub_resource_types.rs new file mode 100644 index 0000000000..88de520094 --- /dev/null +++ b/backend/windmill-common/src/hub_resource_types.rs @@ -0,0 +1,578 @@ +/* + * Author: Ruben Fiszel + * Copyright: Windmill Labs, Inc 2022 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + +//! Keeps the `admins` workspace's resource types, which every workspace falls back to, in +//! step with the hub. Servers run [`spawn_daily_sync`] once a superadmin turns it on, +//! superadmins run [`sync`] on demand, and `windmill cache-rt` bakes the public hub's listing +//! into the image for instances that cannot reach a hub. + +use std::collections::HashMap; + +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; +use serde_json::json; + +use crate::{ + error::{Error, Result}, + global_settings::{ + load_value_from_global_settings, DISABLE_HUB_SETTING, SYNC_HUB_RESOURCE_TYPES_DAILY_SETTING, + }, + utils::{get_license_id_or_uid, HTTP_CLIENT_PERMISSIVE, HUB_API_SECRET}, + worker::HUB_RT_CACHE_DIR, + DB, DEFAULT_HUB_BASE_URL, HUB_BASE_URL, +}; + +const SYNC_INTERVAL: chrono::TimeDelta = chrono::TimeDelta::hours(24); +const RETRY_INTERVAL: chrono::TimeDelta = chrono::TimeDelta::hours(1); +const CHECK_INTERVAL: std::time::Duration = std::time::Duration::from_secs(600); +const SYNC_LOCK_ID: i64 = 737_483_925; +/// `background_task_state.name` of the sync's last attempt and last hub sync. +const SYNC_STATE_TASK: &str = "hub_resource_type_sync"; + +pub fn cache_path() -> String { + format!("{}/resource_types.json", *HUB_RT_CACHE_DIR) +} + +/// A hub resource type with its schema parsed, as synced and as `cache-rt` stores it. +/// +/// `format_extension` and `display_name` are doubly optional: no key (a cache written before +/// the field) leaves the stored value alone, while an explicit null clears it. Plain serde +/// folds both into `None`, hence the wrapping deserializer. +#[derive(Deserialize, Serialize, Clone)] +pub struct HubResourceType { + pub name: String, + pub schema: Option, + pub description: Option, + #[serde( + default, + deserialize_with = "crate::more_serde::double_option", + skip_serializing_if = "Option::is_none" + )] + pub format_extension: Option>, + #[serde( + default, + deserialize_with = "crate::more_serde::double_option", + skip_serializing_if = "Option::is_none" + )] + pub display_name: Option>, + /// Absent from hubs that do not publish filesets, which leaves the stored flag alone. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub is_fileset: Option, +} + +/// An entry of the hub's `/resource_types/list`, whose schema is a JSON string. +#[derive(Deserialize)] +struct HubListing { + name: String, + schema: Option, + description: Option, + /// Absent from hubs predating the column, which do not publish file types either, so + /// the hub is authoritative for it whether or not it sends the key. + #[serde(default)] + format_extension: Option, + #[serde(default, deserialize_with = "crate::more_serde::double_option")] + display_name: Option>, + #[serde(default)] + is_fileset: Option, +} + +/// One hub's listing. A type whose schema does not parse is left out. +pub async fn fetch( + hub_base_url: &str, + api_secret: Option<&str>, + uid: Option<&str>, +) -> Result> { + // Permissive, so `ACCEPT_INVALID_CERTS` lets the sync reach a private hub behind a + // self-signed certificate. + let mut request = HTTP_CLIENT_PERMISSIVE + .get(format!("{hub_base_url}/resource_types/list")) + .header("Accept", "application/json"); + if let Some(uid) = uid { + request = request.header("X-uid", uid); + } + if let Some(secret) = api_secret { + request = request.header("X-api-secret", secret); + } + let response = request.send().await.map_err(|e| { + Error::InternalErr(format!("Failed to reach the hub at {hub_base_url}: {e}")) + })?; + let status = response.status(); + if status == reqwest::StatusCode::UNAUTHORIZED { + return Err(Error::InternalErr(format!( + "The hub at {hub_base_url} refused the request: check the Private Hub API secret" + ))); + } + if !status.is_success() { + return Err(Error::InternalErr(format!( + "The hub at {hub_base_url} answered {status}" + ))); + } + let listing: Vec = response.json().await.map_err(|e| { + Error::InternalErr(format!( + "Failed to parse the resource types listed by {hub_base_url}: {e}" + )) + })?; + + Ok(listing + .into_iter() + .filter_map(|rt| { + let schema = match rt.schema.as_deref().map(serde_json::from_str).transpose() { + Ok(schema) => schema, + Err(e) => { + tracing::warn!( + "Skipping hub resource type {}: invalid schema: {e}", + rt.name + ); + return None; + } + }; + Some(HubResourceType { + name: rt.name, + schema, + description: rt.description, + format_extension: Some(rt.format_extension), + display_name: rt.display_name, + is_fileset: rt.is_fileset, + }) + }) + .collect()) +} + +/// The listing of the hub this instance is configured with. +async fn fetch_configured_hub(db: &DB) -> Result> { + let hub = (**HUB_BASE_URL.load()).clone(); + let uid = get_license_id_or_uid(db).await.ok(); + if hub == DEFAULT_HUB_BASE_URL { + return fetch(&hub, None, uid.as_deref()).await; + } + let secret = (**HUB_API_SECRET.load()).clone(); + let types = fetch(&hub, secret.as_deref(), uid.as_deref()).await?; + if !types.is_empty() { + return Ok(types); + } + // A private hub that publishes no resource types of its own defers to the public one, + // which must not be sent the private hub's secret. + tracing::info!("The hub at {hub} lists no resource types, syncing the public hub's"); + fetch(DEFAULT_HUB_BASE_URL, None, uid.as_deref()).await +} + +#[derive(PartialEq, Debug)] +struct ResourceTypeRow { + schema: Option, + description: Option, + format_extension: Option, + is_fileset: bool, + display_name: Option, +} + +/// The row a synced type should end up as, given what `admins` already stores for it. +fn target_row(rt: &HubResourceType, stored: Option<&ResourceTypeRow>) -> ResourceTypeRow { + let is_fileset = rt + .is_fileset + .unwrap_or_else(|| stored.is_some_and(|s| s.is_fileset)); + // A fileset is a set of files, so it cannot also be one file. Create and update reject + // the pair; this writer bypasses both. + let format_extension = if is_fileset { + None + } else { + match &rt.format_extension { + Some(extension) => extension.clone(), + None => stored.and_then(|s| s.format_extension.clone()), + } + }; + // A name too long for the column counts as absent: one bad entry must not fail the + // upsert and end the rest of the sync. + let display_name = match &rt.display_name { + Some(Some(name)) if name.chars().count() > 100 => { + tracing::warn!( + "Ignoring the display_name of resource type {}: longer than 100 characters", + rt.name + ); + stored.and_then(|s| s.display_name.clone()) + } + Some(name) => name.clone(), + None => stored.and_then(|s| s.display_name.clone()), + }; + ResourceTypeRow { + schema: rt.schema.clone(), + description: rt.description.clone(), + format_extension, + is_fileset, + display_name, + } +} + +#[derive(Clone, Copy, PartialEq, Debug)] +pub enum SyncSource { + Hub, + ImageCache, +} + +pub struct SyncOutcome { + pub source: SyncSource, + /// Every type the source listed, changed or not. + pub listed: Vec, + /// The types this sync created or updated. + pub changed: Vec, +} + +impl SyncOutcome { + pub fn summary(&self) -> String { + let synced = self.changed.len(); + let unchanged = self.listed.len() - synced; + match self.source { + SyncSource::Hub => { + format!("Synced {synced} resource types from the hub ({unchanged} unchanged)") + } + SyncSource::ImageCache => format!( + "Synced {synced} resource types from the image's cache ({unchanged} unchanged)" + ), + } + } +} + +async fn apply(db: &DB, source: SyncSource, types: &[HubResourceType]) -> Result { + let stored: HashMap = sqlx::query_as::< + _, + ( + String, + Option, + Option, + Option, + bool, + Option, + ), + >( + "SELECT name, schema, description, format_extension, is_fileset, display_name + FROM resource_type WHERE workspace_id = 'admins'", + ) + .fetch_all(db) + .await? + .into_iter() + .map( + |(name, schema, description, format_extension, is_fileset, display_name)| { + ( + name, + ResourceTypeRow { schema, description, format_extension, is_fileset, display_name }, + ) + }, + ) + .collect(); + + let mut changed = vec![]; + for rt in types { + let current = stored.get(&rt.name); + let target = target_row(rt, current); + if current == Some(&target) { + continue; + } + sqlx::query( + "INSERT INTO resource_type + (workspace_id, name, schema, description, format_extension, is_fileset, display_name, edited_at) + VALUES ('admins', $1, $2, $3, $4, $5, $6, now()) + ON CONFLICT (workspace_id, name) DO UPDATE + SET schema = EXCLUDED.schema, description = EXCLUDED.description, + format_extension = EXCLUDED.format_extension, is_fileset = EXCLUDED.is_fileset, + display_name = EXCLUDED.display_name, edited_at = now()", + ) + .bind(&rt.name) + .bind(&target.schema) + .bind(&target.description) + .bind(&target.format_extension) + .bind(target.is_fileset) + .bind(&target.display_name) + .execute(db) + .await?; + changed.push(rt.name.clone()); + } + + Ok(SyncOutcome { source, listed: types.iter().map(|rt| rt.name.clone()).collect(), changed }) +} + +#[derive(Deserialize, Default)] +struct SyncState { + last_attempt_at: Option>, + last_hub_sync_at: Option>, +} + +async fn load_state(db: &DB) -> Result { + let value: Option = + sqlx::query_scalar("SELECT value FROM background_task_state WHERE name = $1") + .bind(SYNC_STATE_TASK) + .fetch_optional(db) + .await?; + Ok(value + .and_then(|v| serde_json::from_value(v).ok()) + .unwrap_or_default()) +} + +/// Merged into the stored state, so a writer only ever sets the keys it names. +async fn record_state(db: &DB, patch: serde_json::Value) -> Result<()> { + sqlx::query( + "INSERT INTO background_task_state (name, value) VALUES ($1, $2) + ON CONFLICT (name) DO UPDATE + SET value = background_task_state.value || EXCLUDED.value, updated_at = now()", + ) + .bind(SYNC_STATE_TASK) + .bind(patch) + .execute(db) + .await?; + Ok(()) +} + +async fn setting_enabled(db: &DB, name: &str) -> Result { + Ok(load_value_from_global_settings(db, name) + .await? + .and_then(|v| v.as_bool()) + .unwrap_or(false)) +} + +/// Syncs from the configured hub and records the attempt, successful or not. +async fn sync_from_hub(db: &DB) -> Result { + let attempted_at = Utc::now(); + let result = match fetch_configured_hub(db).await { + Ok(types) => apply(db, SyncSource::Hub, &types).await, + Err(e) => Err(e), + }; + let state = match &result { + Ok(_) => { + json!({ "last_attempt_at": attempted_at, "last_hub_sync_at": attempted_at, "last_error": null }) + } + Err(e) => json!({ "last_attempt_at": attempted_at, "last_error": e.to_string() }), + }; + record_state(db, state).await?; + result +} + +/// Syncs from the configured hub, falling back to the image's cache when the hub is disabled +/// or unreachable and the instance has never synced from a hub. +pub async fn sync(db: &DB) -> Result { + let hub_err = if setting_enabled(db, DISABLE_HUB_SETTING).await? { + Error::BadRequest("The hub is disabled on this instance".to_string()) + } else { + match sync_from_hub(db).await { + Ok(outcome) => return Ok(outcome), + Err(e) => e, + } + }; + match fall_back_to_image_cache(db, &cache_path()).await? { + Some(outcome) => { + tracing::warn!("Hub resource type sync failed, used the image's cache: {hub_err}"); + Ok(outcome) + } + None => Err(hub_err), + } +} + +/// The cache is the listing as of the image's build, so standing in for the hub on an +/// instance that has synced from one would revert every type the hub has updated since. +async fn fall_back_to_image_cache(db: &DB, path: &str) -> Result> { + if let Some(at) = load_state(db).await?.last_hub_sync_at { + tracing::info!( + "Not applying the image's cached resource types: synced from the hub at {at}" + ); + return Ok(None); + } + sync_from_cache_file(db, path).await +} + +/// Applies the listing baked into the image (`SYNC_CACHED_RT`), on every start and whether or +/// not the instance has synced from a hub: it is how an instance that cannot reach the public +/// hub, a private hub listing only its own types included, takes each new image's public types. +/// `None` when the image carries no cache. +pub async fn sync_from_image_cache(db: &DB) -> Result> { + sync_from_cache_file(db, &cache_path()).await +} + +async fn sync_from_cache_file(db: &DB, path: &str) -> Result> { + let content = match tokio::fs::read_to_string(path).await { + Ok(content) => content, + Err(e) if e.kind() == std::io::ErrorKind::NotFound => { + tracing::info!("No cached resource types at {path}"); + return Ok(None); + } + Err(e) => { + return Err(Error::InternalErr(format!( + "Failed to read cached resource types at {path}: {e}" + ))) + } + }; + let types: Vec = serde_json::from_str(&content).map_err(|e| { + Error::InternalErr(format!( + "Failed to parse cached resource types at {path}: {e}" + )) + })?; + Ok(Some(apply(db, SyncSource::ImageCache, &types).await?)) +} + +/// While `sync_hub_resource_types_daily` is on, syncs once a day, from whichever server first +/// finds the sync due. Off by default: each sync overwrites local edits to the `admins` types +/// the hub also defines. The state it reads lives in the database, so servers starting or +/// restarting do not each sync again. +pub fn spawn_daily_sync(db: DB, mut killpill_rx: tokio::sync::broadcast::Receiver<()>) { + tokio::spawn(async move { + // Leaves time for the hub URL and secret settings to load, and spreads out servers + // that start together. + let mut wait = std::time::Duration::from_secs(rand::random_range(30..90)); + loop { + tokio::select! { + _ = tokio::time::sleep(wait) => {} + _ = killpill_rx.recv() => return, + } + if let Err(e) = sync_if_due(&db).await { + tracing::error!("Hub resource type sync: {e:#}"); + } + wait = CHECK_INTERVAL; + } + }); +} + +async fn sync_if_due(db: &DB) -> Result<()> { + if !setting_enabled(db, SYNC_HUB_RESOURCE_TYPES_DAILY_SETTING).await? + || setting_enabled(db, DISABLE_HUB_SETTING).await? + { + return Ok(()); + } + // A transaction-scoped lock: a dropped transaction rolls back and releases it, so a + // pass cut short cannot leave it held on a pooled connection. The transaction only + // owns the lock; the sync runs on other connections. + let mut lock_tx = db.begin().await?; + let locked: bool = sqlx::query_scalar("SELECT pg_try_advisory_xact_lock($1)") + .bind(SYNC_LOCK_ID) + .fetch_one(&mut *lock_tx) + .await?; + if !locked { + return Ok(()); + } + let state = load_state(db).await?; + let now = Utc::now(); + let due = state + .last_hub_sync_at + .is_none_or(|at| now - at >= SYNC_INTERVAL) + && state + .last_attempt_at + .is_none_or(|at| now - at >= RETRY_INTERVAL); + // Hub only. Falling back to the image's cache here would re-apply the same listing every + // hour on an instance that never reaches a hub, over any edit made since; the cache is + // applied at boot (`SYNC_CACHED_RT`) or on demand instead. + if due { + match sync_from_hub(db).await { + Ok(outcome) if outcome.changed.is_empty() => tracing::info!("{}", outcome.summary()), + Ok(outcome) => { + tracing::info!("{}: {}", outcome.summary(), outcome.changed.join(", ")) + } + Err(e) => tracing::warn!("Hub resource type sync failed, retrying in an hour: {e}"), + } + } + lock_tx.rollback().await?; + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn hub_type( + format_extension: Option>, + display_name: Option>, + is_fileset: Option, + ) -> HubResourceType { + HubResourceType { + name: "t".to_string(), + schema: None, + description: None, + format_extension: format_extension.map(|e| e.map(str::to_string)), + display_name: display_name.map(|n| n.map(str::to_string)), + is_fileset, + } + } + + fn stored( + format_extension: Option<&str>, + display_name: Option<&str>, + is_fileset: bool, + ) -> ResourceTypeRow { + ResourceTypeRow { + schema: None, + description: None, + format_extension: format_extension.map(str::to_string), + is_fileset, + display_name: display_name.map(str::to_string), + } + } + + #[test] + fn absent_keys_keep_the_stored_values_and_nulls_clear_them() { + let current = stored(Some("csv"), Some("T"), false); + let kept = target_row(&hub_type(None, None, None), Some(¤t)); + assert_eq!(kept, current); + let cleared = target_row(&hub_type(Some(None), Some(None), None), Some(¤t)); + assert_eq!(cleared, stored(None, None, false)); + } + + #[test] + fn a_fileset_never_takes_a_format_extension() { + let local_fileset = stored(None, None, true); + let row = target_row( + &hub_type(Some(Some("csv")), None, None), + Some(&local_fileset), + ); + assert_eq!(row, stored(None, None, true)); + let row = target_row(&hub_type(Some(Some("csv")), None, Some(true)), None); + assert_eq!(row, stored(None, None, true)); + } + + #[sqlx::test(migrations = "../migrations")] + async fn only_sync_cached_rt_applies_the_image_cache_after_a_hub_sync( + db: DB, + ) -> anyhow::Result<()> { + let dir = tempfile::tempdir()?; + let path = dir.path().join("resource_types.json"); + std::fs::write( + &path, + json!([{ "name": "cached_type", "schema": null, "description": "image copy" }]) + .to_string(), + )?; + let path = path.to_str().unwrap(); + let description = || { + sqlx::query_scalar::<_, Option>( + "SELECT description FROM resource_type WHERE workspace_id = 'admins' AND name = 'cached_type'", + ) + .fetch_one(&db) + }; + + assert!(fall_back_to_image_cache(&db, path).await?.is_some()); + assert_eq!(description().await?.as_deref(), Some("image copy")); + + record_state(&db, json!({ "last_hub_sync_at": Utc::now() })).await?; + sqlx::query( + "UPDATE resource_type SET description = 'hub copy' WHERE workspace_id = 'admins' AND name = 'cached_type'", + ) + .execute(&db) + .await?; + assert!(fall_back_to_image_cache(&db, path).await?.is_none()); + assert_eq!(description().await?.as_deref(), Some("hub copy")); + + assert!(sync_from_cache_file(&db, path).await?.is_some()); + assert_eq!(description().await?.as_deref(), Some("image copy")); + Ok(()) + } + + #[sqlx::test(migrations = "../migrations")] + async fn no_daily_sync_unless_turned_on(db: DB) -> anyhow::Result<()> { + sync_if_due(&db).await?; + + let state: Option = + sqlx::query_scalar("SELECT value FROM background_task_state WHERE name = $1") + .bind(SYNC_STATE_TASK) + .fetch_optional(&db) + .await?; + assert_eq!(state, None); + Ok(()) + } +} diff --git a/backend/windmill-common/src/instance_config.rs b/backend/windmill-common/src/instance_config.rs index fd790547b6..7c76fab2ee 100644 --- a/backend/windmill-common/src/instance_config.rs +++ b/backend/windmill-common/src/instance_config.rs @@ -267,6 +267,8 @@ pub struct GlobalSettings { #[serde(skip_serializing_if = "Option::is_none")] pub disable_hub: Option, #[serde(skip_serializing_if = "Option::is_none")] + pub sync_hub_resource_types_daily: Option, + #[serde(skip_serializing_if = "Option::is_none")] pub auto_build_binary_on_deploy: Option, #[serde(skip_serializing_if = "Option::is_none")] pub ai_sessions_instance_storage_fallback: Option, @@ -275,10 +277,14 @@ pub struct GlobalSettings { #[serde(skip_serializing_if = "Option::is_none")] pub auto_build_binary_tag: Option, #[serde(skip_serializing_if = "Option::is_none")] + pub dependency_job_tag: Option, + #[serde(skip_serializing_if = "Option::is_none")] pub ws_base_url: Option, #[serde(skip_serializing_if = "Option::is_none")] pub github_app_webhook_base_url: Option, #[serde(skip_serializing_if = "Option::is_none")] + pub api_base_url: Option, + #[serde(skip_serializing_if = "Option::is_none")] pub email_domain: Option, #[serde(skip_serializing_if = "Option::is_none")] pub hub_base_url: Option, @@ -961,6 +967,9 @@ pub struct WorkerGroupConfig { pub autoscaling: Option, #[serde(skip_serializing_if = "Option::is_none")] pub native_mode: Option, + /// Workers of the group pull no new job while this is set; queued jobs wait. + #[serde(skip_serializing_if = "Option::is_none")] + pub paused: Option, /// Object store this group's dependency cache uses instead of the instance one. Same shape /// as the instance `object_store_cache_config`. #[serde(skip_serializing_if = "Option::is_none")] @@ -1006,6 +1015,43 @@ pub struct ConfigsDiff { pub deletes: Vec, } +/// Told of each write an instance config apply makes, as soon as it has landed: the writes +/// are not transactional, so an apply that fails midway must not leave the ones it did make +/// unrecorded. The audit entry is the implementor's to write, as this crate sits below +/// `windmill-audit`. +pub trait ConfigWriteObserver: Sync { + /// `value` is `None` for a deleted setting. + fn setting_written( + &self, + key: &str, + value: Option<&serde_json::Value>, + ) -> impl std::future::Future> + Send; + + fn worker_config_written( + &self, + group: &str, + deleted: bool, + ) -> impl std::future::Future> + Send; +} + +impl ConfigWriteObserver for () { + async fn setting_written( + &self, + _key: &str, + _value: Option<&serde_json::Value>, + ) -> crate::error::Result<()> { + Ok(()) + } + + async fn worker_config_written( + &self, + _group: &str, + _deleted: bool, + ) -> crate::error::Result<()> { + Ok(()) + } +} + /// Settings that must never be deleted by the operator or bulk API. pub const PROTECTED_SETTINGS: &[&str] = &[ "ducklake_user_pg_pwd", @@ -1017,12 +1063,44 @@ pub const PROTECTED_SETTINGS: &[&str] = &[ "rsa_keys", "jwt_secret", "min_keep_alive_version", + // Deleting it would let a worker sign OIDC tokens with a local key again. + "oidc_external_signer", ]; +/// Secrets the server signs with or generated for itself. `jwt_secret` signs API and job +/// tokens and `rsa_keys` signs job OIDC tokens, so reading either is enough to mint tokens +/// for any user. Superadmin reads return them only while `server_secrets_exported()`. +/// Withholding them from reads does not make them unwritable: the two signing keys are not +/// in `HIDDEN_SETTINGS`, so config can still set them (an operator ConfigMap may set +/// `jwt_secret`). +pub const SERVER_SECRET_SETTINGS: &[&str] = &[ + "jwt_secret", + "rsa_keys", + "custom_instance_replication_pwd", + "external_instance_pg_state", +]; + +/// Whether superadmin API reads (settings list, single-setting read, config export) return +/// `SERVER_SECRET_SETTINGS`. On by default so `wmill instance get-config` and `pull` carry a +/// full migration; `EXPORT_SERVER_SECRETS=false` withholds them from every API response, so a +/// leaked superadmin token cannot take the signing keys. Fails closed: once set to a +/// non-empty value, only `true`/`1`/`yes`/`on` keep exporting, so `0` or a typo withholds. +/// Read per call: the flag is cheap and tests flip it in-process. +pub fn server_secrets_exported() -> bool { + match std::env::var("EXPORT_SERVER_SECRETS") { + Ok(v) if !v.trim().is_empty() => matches!( + v.trim().to_ascii_lowercase().as_str(), + "true" | "1" | "yes" | "on" + ), + _ => true, + } +} + +pub fn is_withheld_server_secret(name: &str) -> bool { + SERVER_SECRET_SETTINGS.contains(&name) && !server_secrets_exported() +} + /// Internal settings that are never exposed via the API or included in config exports. -/// Note: jwt_secret is intentionally NOT hidden — it is included in YAML exports so that -/// operators can set it via ConfigMap. It is protected from deletion (PROTECTED_SETTINGS) -/// and from being set to empty/null, and its value is partially redacted in log output. pub const HIDDEN_SETTINGS: &[&str] = &[ "uid", "min_keep_alive_version", @@ -1040,6 +1118,8 @@ pub const HIDDEN_SETTINGS: &[&str] = &[ "custom_instance_replication_pwd", // Same for the passwords and database registry Windmill keeps for the external cluster. "external_instance_pg_state", + // Written by servers that sign OIDC tokens with an external signer. + "oidc_external_signer", ]; /// Top-level settings whose entire value is sensitive and must be fully redacted in logs. @@ -1340,8 +1420,8 @@ pub fn diff_worker_configs( } /// Declaratively replace the global settings, rejecting a -/// `github_app_webhook_base_url` or `http_route_default_allowed_origins` the -/// API would reject. +/// `github_app_webhook_base_url`, `api_base_url` or +/// `http_route_default_allowed_origins` the API would reject. /// /// Every declarative writer (the `sync-config` CLI, the Kubernetes operator's /// ConfigMap sync) MUST go through this rather than calling @@ -1359,34 +1439,44 @@ pub fn diff_worker_configs( /// AUTHORIZATION: replaces instance-wide settings and takes no authed context, so /// callers MUST have established superadmin or equivalent system authority (the CLI /// and the operator both run with direct instance credentials). +/// +/// `observer` is what audits the sync: no user request stands behind these writes, so +/// nothing else records them. pub async fn sync_global_settings_declarative( db: &sqlx::Pool, current: &BTreeMap, desired: &BTreeMap, + observer: &impl ConfigWriteObserver, ) -> anyhow::Result<()> { - let webhook_key = crate::global_settings::GITHUB_APP_WEBHOOK_BASE_URL_SETTING; - // Non-string shapes are rejected rather than ignored: `as_str()` alone would let a - // bool/number/object through as if the key were absent, and the diff below would - // then persist it — where the HTTP path answers "must be a URL string". - match desired.get(webhook_key) { - None | Some(serde_json::Value::Null) => {} - Some(serde_json::Value::String(s)) if s.trim().is_empty() => {} - Some(serde_json::Value::String(s)) => crate::global_settings::validate_webhook_base_url(s) - .map_err(|e| anyhow::anyhow!("{webhook_key}: {e}"))?, - // Names the JSON kind rather than printing it: this is the last message on - // this path that could report submitted content, and an object or array could - // carry a secret into `sync-config` output and operator logs. - Some(other) => { - let kind = match other { - serde_json::Value::Bool(_) => "a boolean", - serde_json::Value::Number(_) => "a number", - serde_json::Value::Array(_) => "an array", - serde_json::Value::Object(_) => "an object", - _ => "a non-string value", - }; - return Err(anyhow::anyhow!( - "{webhook_key} must be a URL string, got {kind}" - )); + for url_key in [ + crate::global_settings::GITHUB_APP_WEBHOOK_BASE_URL_SETTING, + crate::global_settings::API_BASE_URL_SETTING, + ] { + // Non-string shapes are rejected rather than ignored: `as_str()` alone would let a + // bool/number/object through as if the key were absent, and the diff below would + // then persist it — where the HTTP path answers "must be a URL string". + match desired.get(url_key) { + None | Some(serde_json::Value::Null) => {} + Some(serde_json::Value::String(s)) if s.trim().is_empty() => {} + Some(serde_json::Value::String(s)) => { + crate::global_settings::validate_webhook_base_url(s) + .map_err(|e| anyhow::anyhow!("{url_key}: {e}"))? + } + // Names the JSON kind rather than printing it: this is the last message on + // this path that could report submitted content, and an object or array could + // carry a secret into `sync-config` output and operator logs. + Some(other) => { + let kind = match other { + serde_json::Value::Bool(_) => "a boolean", + serde_json::Value::Number(_) => "a number", + serde_json::Value::Array(_) => "an array", + serde_json::Value::Object(_) => "an object", + _ => "a non-string value", + }; + return Err(anyhow::anyhow!( + "{url_key} must be a URL string, got {kind}" + )); + } } } @@ -1420,16 +1510,22 @@ pub async fn sync_global_settings_declarative( .map_err(|e| anyhow::anyhow!("{max_expiration_key}: {e}"))?; let mut diff = diff_global_settings(current, desired, ApplyMode::Replace); - crate::external_instance_pg::write_external_instance_pg_from_diff(db, &mut diff).await?; - apply_settings_diff(db, &diff).await?; + crate::external_instance_pg::write_external_instance_pg_from_diff(db, &mut diff, observer) + .await?; + apply_settings_diff(db, &diff, observer).await?; Ok(()) } -/// Apply a settings diff to the global_settings table. +/// Apply a settings diff to the global_settings table, reporting each write to `observer`. +/// +/// AUTHORIZATION: writes instance-wide settings, credentials included, and checks +/// nothing; `observer` records and authorizes nothing. Callers MUST have established +/// superadmin or equivalent system authority. pub async fn apply_settings_diff( db: &sqlx::Pool, diff: &SettingsDiff, + observer: &impl ConfigWriteObserver, ) -> anyhow::Result<()> { for (key, value) in &diff.upserts { sqlx::query( @@ -1452,6 +1548,7 @@ pub async fn apply_settings_diff( format_setting_value(key, value) ); } + observer.setting_written(key, Some(value)).await?; } for key in &diff.deletes { sqlx::query("DELETE FROM global_settings WHERE name = $1") @@ -1466,6 +1563,7 @@ pub async fn apply_settings_diff( } else { tracing::info!("Deleted global setting: {key}"); } + observer.setting_written(key, None).await?; } if diff.unchanged_count > 0 { tracing::info!("{} global setting(s) unchanged", diff.unchanged_count); @@ -1473,10 +1571,16 @@ pub async fn apply_settings_diff( Ok(()) } -/// Apply a configs diff to the config table (worker__ prefix). +/// Apply a configs diff to the config table (worker__ prefix), reporting each write to +/// `observer`. +/// +/// AUTHORIZATION: writes worker group configs, whose init scripts run on the workers, and +/// checks nothing; `observer` records and authorizes nothing. Callers MUST have +/// established superadmin or equivalent system authority. pub async fn apply_configs_diff( db: &sqlx::Pool, diff: &ConfigsDiff, + observer: &impl ConfigWriteObserver, ) -> anyhow::Result<()> { for (group_name, config_value) in &diff.upserts { let db_key = format!("worker__{group_name}"); @@ -1489,6 +1593,7 @@ pub async fn apply_configs_diff( .execute(db) .await?; tracing::info!("Synced worker config: {db_key}"); + observer.worker_config_written(group_name, false).await?; } for group_name in &diff.deletes { let db_key = format!("worker__{group_name}"); @@ -1497,6 +1602,7 @@ pub async fn apply_configs_diff( .execute(db) .await?; tracing::info!("Deleted worker config: {db_key}"); + observer.worker_config_written(group_name, true).await?; } Ok(()) } @@ -1567,6 +1673,21 @@ pub fn resolve_env_refs(settings: &mut GlobalSettings) -> Result<(), String> { impl InstanceConfig { /// Read the full instance configuration from the database. pub async fn from_db(db: &sqlx::Pool) -> anyhow::Result { + Self::read_from_db(db, true).await + } + + /// The instance configuration as an API response may carry it: `from_db` without + /// `SERVER_SECRET_SETTINGS` unless `server_secrets_exported()`. It holds admin-entered + /// credentials (`license_key`, `scim_token`, SMTP and OAuth secrets), so callers must + /// have checked superadmin. + pub async fn from_db_for_api(db: &sqlx::Pool) -> anyhow::Result { + Self::read_from_db(db, server_secrets_exported()).await + } + + async fn read_from_db( + db: &sqlx::Pool, + include_server_secrets: bool, + ) -> anyhow::Result { // Read global_settings table → flat map → deserialize into GlobalSettings let rows: Vec<(String, serde_json::Value)> = sqlx::query_as("SELECT name, value FROM global_settings") @@ -1575,6 +1696,9 @@ impl InstanceConfig { let map: serde_json::Map = rows .into_iter() .filter(|(name, _)| !HIDDEN_SETTINGS.contains(&name.as_str())) + .filter(|(name, _)| { + include_server_secrets || !SERVER_SECRET_SETTINGS.contains(&name.as_str()) + }) .collect(); let global_settings: GlobalSettings = serde_json::from_value(serde_json::Value::Object(map))?; @@ -1600,7 +1724,11 @@ impl InstanceConfig { /// /// Reads the current state, computes a diff, then applies upserts and /// deletes for both global settings and worker configs. - pub async fn sync_to_db(&self, db: &sqlx::Pool) -> anyhow::Result<()> { + pub async fn sync_to_db( + &self, + db: &sqlx::Pool, + observer: &impl ConfigWriteObserver, + ) -> anyhow::Result<()> { let desired_settings = self.global_settings.to_settings_map(); // Current global settings @@ -1610,7 +1738,8 @@ impl InstanceConfig { .await?; let current_settings: BTreeMap = rows.into_iter().collect(); - sync_global_settings_declarative(db, ¤t_settings, &desired_settings).await?; + sync_global_settings_declarative(db, ¤t_settings, &desired_settings, observer) + .await?; // Worker configs let desired_configs: BTreeMap = self @@ -1638,7 +1767,7 @@ impl InstanceConfig { let configs_diff = diff_worker_configs(¤t_configs, &desired_configs, ApplyMode::Replace); - apply_configs_diff(db, &configs_diff).await?; + apply_configs_diff(db, &configs_diff, observer).await?; Ok(()) } diff --git a/backend/windmill-common/src/item_digest.rs b/backend/windmill-common/src/item_digest.rs new file mode 100644 index 0000000000..23110bcae7 --- /dev/null +++ b/backend/windmill-common/src/item_digest.rs @@ -0,0 +1,132 @@ +//! Content digests of deployed scripts and flows, reproducible with `wmill digest` +//! (`cli/src/commands/digest/digest.ts`) from a `wmill sync` checkout pulled after the +//! deployment's dependency jobs: they write the locks, and store loops and branches with +//! their defaults filled in. The two implementations must hash the same bytes: change one +//! only together with the other. Numbers hash as the doubles they parse to, which needs +//! serde_json's `float_roundtrip` to parse them as exactly as JavaScript does. +//! +//! A digest is the lowercase hex SHA-256 of the canonical JSON (RFC 8785) of the item, with +//! every object member whose value is null removed: +//! - a script: `{"content", "lock", "modules", "codebase"}`, where `modules` is the +//! multi-file script's `{: {"content", "lock"}}` and is left out when empty. +//! The language is not covered: a checkout only records it in the file extension; +//! - a flow step's inline script: `{"content", "lock"}`; +//! - a flow: its `value`. A flow whose value references code in `flow_node` has none, see +//! [`references_flow_nodes`]. + +use serde_json::Value; +use sha2::{Digest, Sha256}; + +use crate::flows::{Branch, FlowModule, FlowModuleValue, FlowValue, ToolValue}; + +pub fn digest(value: &Value) -> String { + let mut out = String::new(); + write_canonical(value, &mut out); + format!("{:x}", Sha256::digest(out.as_bytes())) +} + +pub fn canonical_json(value: &Value) -> String { + let mut out = String::new(); + write_canonical(value, &mut out); + out +} + +fn write_canonical(value: &Value, out: &mut String) { + match value { + Value::Null => out.push_str("null"), + Value::Bool(b) => out.push_str(if *b { "true" } else { "false" }), + // RFC 8785 serializes every number as an IEEE double, the way ECMAScript prints it. + Value::Number(n) => { + let f = n.as_f64().unwrap_or(0.0); + out.push_str(ryu_js::Buffer::new().format(f)) + } + Value::String(s) => out.push_str(&serde_json::to_string(s).unwrap_or_default()), + Value::Array(items) => { + out.push('['); + for (i, item) in items.iter().enumerate() { + if i > 0 { + out.push(','); + } + write_canonical(item, out); + } + out.push(']'); + } + Value::Object(members) => { + // RFC 8785 orders members by the UTF-16 code units of their names. + let mut members: Vec<_> = members.iter().filter(|(_, v)| !v.is_null()).collect(); + members.sort_by(|(a, _), (b, _)| a.encode_utf16().cmp(b.encode_utf16())); + out.push('{'); + for (i, (k, v)) in members.into_iter().enumerate() { + if i > 0 { + out.push(','); + } + out.push_str(&serde_json::to_string(k).unwrap_or_default()); + out.push(':'); + write_canonical(v, out); + } + out.push('}'); + } + } +} + +/// Whether a flow value runs code stored outside it, in `flow_node` rows: an inline script +/// step `{"type": "flowscript", "id"}`, or a loop's or branch's steps in a `*_node`. Its digest +/// would not cover that code, and a checkout cannot reproduce it either, since the export +/// carries the references. A value or step that does not parse runs nothing, so it counts as +/// none. +pub fn references_flow_nodes(value: &Value) -> bool { + let Ok(flow) = serde_json::from_value::(value.clone()) else { + return false; + }; + flow.modules + .iter() + .chain(flow.preprocessor_module.as_deref()) + .chain(flow.failure_module.as_deref()) + .any(module_references_flow_nodes) +} + +fn module_references_flow_nodes(module: &FlowModule) -> bool { + module + .get_value() + .is_ok_and(|v| module_value_references_flow_nodes(&v)) +} + +fn module_value_references_flow_nodes(value: &FlowModuleValue) -> bool { + let any = |modules: &[FlowModule]| modules.iter().any(module_references_flow_nodes); + let branches = |branches: &[Branch]| { + branches + .iter() + .any(|b| b.modules_node.is_some() || any(&b.modules)) + }; + match value { + FlowModuleValue::FlowScript { .. } => true, + FlowModuleValue::ForloopFlow { modules, modules_node, .. } + | FlowModuleValue::WhileloopFlow { modules, modules_node, .. } => { + modules_node.is_some() || any(modules) + } + FlowModuleValue::BranchOne { branches: b, default, default_node } => { + default_node.is_some() || any(default) || branches(b) + } + FlowModuleValue::BranchAll { branches: b, .. } => branches(b), + FlowModuleValue::AIAgent { tools, .. } => tools.iter().any(|tool| match &tool.value { + ToolValue::FlowModule(v) => module_value_references_flow_nodes(v), + _ => false, + }), + _ => false, + } +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + #[test] + fn canonical_form_follows_rfc_8785_without_nulls() { + let v = json!({"b": [1.0, 1e21, 0.1, -0.0, null], "a": null, "é": "\u{1f}\"\n", "\u{e000}": 1, "\u{1f600}": 2}); + assert_eq!( + canonical_json(&v), + "{\"b\":[1,1e+21,0.1,0,null],\"é\":\"\\u001f\\\"\\n\",\"\u{1f600}\":2,\"\u{e000}\":1}" + ); + } +} diff --git a/backend/windmill-common/src/job_provenance.rs b/backend/windmill-common/src/job_provenance.rs index 63075b0c37..96db3b8a87 100644 --- a/backend/windmill-common/src/job_provenance.rs +++ b/backend/windmill-common/src/job_provenance.rs @@ -1,6 +1,7 @@ +use serde_json::Value; use uuid::Uuid; -use crate::{db::DB, error::Result, jobs::JobKind, scripts::ScriptHash}; +use crate::{db::DB, error::Result, item_digest, jobs::JobKind, scripts::ScriptHash}; /// Where a job's code comes from, as far as its chain of parents can prove it. pub struct JobProvenance { @@ -28,7 +29,39 @@ pub struct JobProvenance { pub root_path: Option, pub root_kind: JobKind, pub root_trigger_kind: Option, + /// What triggered the root job, as recorded in `v2_job.trigger`. + pub root_trigger: Option, pub root_version: Option, + /// See [`item_digest`]. Only scripts, flows and flow steps' inline scripts have one. + pub digest: Option, + pub root_digest: Option, + /// The job's worker tag. + pub tag: String, + /// The group of the worker running the job, if it is running. + pub worker_group: Option, + pub run_as_type: RunAsType, +} + +/// What `permissioned_as` names. +#[derive(Debug, PartialEq, Eq, Clone, Copy)] +pub enum RunAsType { + User, + ServiceAccount, + Group, + /// No member of the workspace: a superadmin outside it, a built-in identity + /// (`superadmin_secret@windmill.dev`, ...) or a user since removed. + NonMember, +} + +impl RunAsType { + pub fn as_str(&self) -> &'static str { + match self { + RunAsType::User => "user", + RunAsType::ServiceAccount => "service_account", + RunAsType::Group => "group", + RunAsType::NonMember => "non_member", + } + } } #[derive(Debug, PartialEq, Eq)] @@ -58,6 +91,8 @@ struct LineageJob { runnable_path: Option, runnable_id: Option, trigger_kind: Option, + trigger: Option, + tag: String, permissioned_as: String, origin_verified: bool, current_version: bool, @@ -66,6 +101,10 @@ struct LineageJob { restarted_from: Option, app_stamped: bool, args_modules: bool, + worker_group: Option, + run_as_type: Option, + /// What the digest hashes, for the job and the root only. + digest_source: Option, } /// Reads any job of `w_id` regardless of the caller: authorize access to `job_id` first. @@ -74,16 +113,17 @@ pub async fn job_provenance(db: &DB, job_id: &Uuid, w_id: &str) -> Result