diff --git a/.github/workflows/ai-evals-scheduled.yml b/.github/workflows/ai-evals-scheduled.yml index cc1c0d6763..1284154a91 100644 --- a/.github/workflows/ai-evals-scheduled.yml +++ b/.github/workflows/ai-evals-scheduled.yml @@ -1,25 +1,22 @@ name: AI Evals (scheduled) -# Full ai_evals suites on current models, posted to the AI evals dashboard in -# the windmill-prod workspace (f/ai/ai_evals_dashboard) so quality is tracked -# over time. Weekly, since one pass of every suite at --runs 3 costs about 50M -# tokens per model: every suite runs on MODELS, and global (the mode users get) -# also runs on GLOBAL_EXTRA_MODELS, one flagship per other provider. Run it by -# hand to measure a branch against main. +# The global (AI chat) and cli suites on current models, posted to the AI evals +# dashboard in the windmill-prod workspace (f/ai/ai_evals_dashboard) so quality is +# tracked over time. Weekly: one global pass at --runs 3 is about 45M tokens per +# model. Each mode and model is its own job, so a slow model neither delays nor +# shares a timeout with the others. Run it by hand to measure a branch against +# main, or to include the flow, app and script suites. on: schedule: - cron: "0 3 * * 1" workflow_dispatch: inputs: modes: - description: "Space-separated modes" - default: "global flow app script cli" + description: "Space-separated modes (global, cli, flow, app, script)" + default: "global cli" models: - description: "Space-separated model aliases (bun run cli -- models)" - default: "sonnet-5.5" - global_extra_models: - description: "Extra model aliases for global mode only" - default: "gpt-6-astra gemini-3.8-flash" + description: "Space-separated model aliases (bun run cli -- models); cli runs only the Claude ones" + default: "sonnet-5.5 opus-5.5 gpt-6-sol gpt-6-astra deepseek-v4-pro" runs: description: "Runs per case" default: "3" @@ -31,8 +28,7 @@ concurrency: group: ai-evals-scheduled-${{ github.ref }} env: - MODELS: ${{ inputs.models || 'sonnet-5.5' }} - GLOBAL_EXTRA_MODELS: ${{ inputs.global_extra_models || 'gpt-6-astra gemini-3.8-flash' }} + MODELS: ${{ inputs.models || 'sonnet-5.5 opus-5.5 gpt-6-sol gpt-6-astra deepseek-v4-pro' }} RUNS: ${{ inputs.runs || '3' }} REASONING: ${{ inputs.reasoning }} # Routes on the cloud instance are prefixed with their workspace. @@ -42,14 +38,20 @@ jobs: setup: runs-on: ubuntu-latest outputs: - modes: ${{ steps.modes.outputs.modes }} + matrix: ${{ steps.matrix.outputs.matrix }} steps: - - id: modes + - id: matrix env: - MODES: ${{ inputs.modes || 'global flow app script cli' }} + MODES: ${{ inputs.modes || 'global cli' }} + # cli drives the Claude Agent SDK, so it only takes the Claude aliases + # (`cli` entries in ai_evals/core/models.ts). run: | - echo "modes=$(jq -cn --arg m "$MODES" '$m | split(" ") - | map(select(IN("global", "flow", "app", "script", "cli")))')" >> "$GITHUB_OUTPUT" + echo "matrix=$(jq -cn --arg modes "$MODES" --arg models "$MODELS" ' + def words: split(" ") | map(select(. != "")); + [ $modes | words[] | select(IN("global", "flow", "app", "script", "cli")) as $mode + | $models | words[] + | select($mode != "cli" or test("^(claude-)?(haiku|sonnet|opus)"; "i")) + | {mode: $mode, model: .} ]')" >> "$GITHUB_OUTPUT" evals: needs: setup @@ -58,7 +60,7 @@ jobs: strategy: fail-fast: false matrix: - mode: ${{ fromJSON(needs.setup.outputs.modes) }} + include: ${{ fromJSON(needs.setup.outputs.matrix) }} services: postgres: image: postgres:16 @@ -118,10 +120,11 @@ jobs: working-directory: ./cli run: bun install && ./gen_wm_client.sh && ./windmill-utils-internal/gen_wm_client.sh - - name: Run ${{ matrix.mode }} evals + - name: Run ${{ matrix.mode }} evals on ${{ matrix.model }} working-directory: ./ai_evals env: MODE: ${{ matrix.mode }} + MODEL: ${{ matrix.model }} WMILL_AI_EVAL_BACKEND_URL: http://localhost:8000 WMILL_AI_EVAL_BACKEND_WORKSPACE: integration-tests ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} @@ -131,14 +134,10 @@ jobs: run: | bun install mkdir -p results - models="$MODELS" - [ "$MODE" = global ] && models="$models $GLOBAL_EXTRA_MODELS" reasoning=() [ -n "$REASONING" ] && [ "$MODE" != cli ] && reasoning=(--reasoning "$REASONING") - for m in $models; do - bun run cli -- run "$MODE" --model "$m" --runs "$RUNS" "${reasoning[@]}" \ - --output "$PWD/results/$MODE-$m.json" || echo "::warning::$MODE on $m errored" - done + bun run cli -- run "$MODE" --model "$MODEL" --runs "$RUNS" "${reasoning[@]}" \ + --output "$PWD/results/$MODE-$MODEL.json" || echo "::warning::$MODE on $MODEL errored" - name: Post results to the dashboard if: always() @@ -166,7 +165,7 @@ jobs: uses: actions/upload-artifact@v4 if: always() with: - name: ai-evals-${{ matrix.mode }} + name: ai-evals-${{ matrix.mode }}-${{ matrix.model }} path: | ai_evals/logs ai_evals/results diff --git a/.release-please-manifest.json b/.release-please-manifest.json index afbaa8e7b0..2c9a3730b3 100644 --- a/.release-please-manifest.json +++ b/.release-please-manifest.json @@ -1,3 +1,3 @@ { - ".": "1.821.0" + ".": "1.825.0" } diff --git a/AGENTS.md b/AGENTS.md index 8451a0a600..e4915cdcb6 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -90,6 +90,12 @@ are for a plain checkout with nothing running. - **Login**: `admin@windmill.dev` / `changeme` - **Instance settings**: navigate to `/#superadmin-settings` - **Migrations**: use `cargo sqlx migrate add -r ` from `backend/` to create new migrations (never generate timestamps manually) +- **Resource types ship through the hub, not migrations.** A new type, a provider's credential + included, is a `hub//.resource_type.json` in `windmill-labs/windmill-integrations` + and reaches instances by hub sync; a product name its identifier misses (`typesafe` → + "TypeSafe") is that file's `display_name`, not a word in `resourceTypeDisplay.ts`'s map. Only a + type Windmill itself defines and reads as a product object (`ai_skill`, `ai_instruction`) is + also seeded by a migration, so the feature works before a sync. For local testing, insert the row. ### Per-worktree ports and database diff --git a/CHANGELOG.md b/CHANGELOG.md index 8aa66a785d..09cccb5fcf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,110 @@ # Changelog +## [1.825.0](https://github.com/windmill-labs/windmill/compare/v1.824.1...v1.825.0) (2026-10-06) + + +### Features + +* choose build with AI or code when creating a raw app ([#11560](https://github.com/windmill-labs/windmill/issues/11560)) ([431f861](https://github.com/windmill-labs/windmill/commit/431f86138815fce432e282e8ac495953c5ecf98a)) +* confine jobs:read to the runnable paths it names ([#11550](https://github.com/windmill-labs/windmill/issues/11550)) ([b371af3](https://github.com/windmill-labs/windmill/commit/b371af31c96e6890fe9223233d2c6e53ddea67d4)) +* open a run as a pushed page on the narrow runs page ([#11541](https://github.com/windmill-labs/windmill/issues/11541)) ([fd8de45](https://github.com/windmill-labs/windmill/commit/fd8de45cc7cfcba2c374dd86f87f4d3e8aec6ded)) +* say a canceled team plan is canceled, not past due ([#11539](https://github.com/windmill-labs/windmill/issues/11539)) ([18e44d3](https://github.com/windmill-labs/windmill/commit/18e44d31742b8d16852b8c6ae663979666e064d7)) + + +### Bug Fixes + +* group raw app edits that carry a file diff in the ai chat ([#11558](https://github.com/windmill-labs/windmill/issues/11558)) ([5912d30](https://github.com/windmill-labs/windmill/commit/5912d300a550a33674fd73740d613d1b434fbba1)) +* honor custom timeout in setup phases and name the limit hit ([#11557](https://github.com/windmill-labs/windmill/issues/11557)) ([4fd4f76](https://github.com/windmill-labs/windmill/commit/4fd4f76d0df740cb3eebed1a0625b970484807fc)) +* search slow runs filters by time windows instead of timing out ([#11549](https://github.com/windmill-labs/windmill/issues/11549)) ([2bf9cfc](https://github.com/windmill-labs/windmill/commit/2bf9cfcd59e1101695160524c31a4da1a41699dc)) +* serve the raw-app preview shell sandboxed, load a blob copy in the editor ([#11554](https://github.com/windmill-labs/windmill/issues/11554)) ([5f92daa](https://github.com/windmill-labs/windmill/commit/5f92daab6d0ecac30dd384e1b0f302af13fdd9ab)) +* simplify the run page AI card to a prompt and a chat button ([#11561](https://github.com/windmill-labs/windmill/issues/11561)) ([8c74f75](https://github.com/windmill-labs/windmill/commit/8c74f75de3fa62b07186ca4f56dae4a7e4758034)) + +## [1.824.1](https://github.com/windmill-labs/windmill/compare/v1.824.0...v1.824.1) (2026-10-05) + + +### Bug Fixes + +* keep monaco's diff engine out of the app shell bundle ([#11542](https://github.com/windmill-labs/windmill/issues/11542)) ([0908dee](https://github.com/windmill-labs/windmill/commit/0908dee897e0f87b97418a23ede3662d96270340)) + +## [1.824.0](https://github.com/windmill-labs/windmill/compare/v1.823.1...v1.824.0) (2026-10-05) + + +### Features + +* add an AI decision flow step (TypeSafe's Jev, Cloudflare's Clef) ([#11474](https://github.com/windmill-labs/windmill/issues/11474)) ([3cba8a7](https://github.com/windmill-labs/windmill/commit/3cba8a7e4cca3ef20ba73a7dad79d892bdc9fed8)) +* add notion oauth provider ([#11528](https://github.com/windmill-labs/windmill/issues/11528)) ([8b2dc1e](https://github.com/windmill-labs/windmill/commit/8b2dc1e46c7f4e3859263f18b686ce6f3ce6fd30)) +* **ai-chat:** show per-file diffs for raw app edits ([#11521](https://github.com/windmill-labs/windmill/issues/11521)) ([ec0883a](https://github.com/windmill-labs/windmill/commit/ec0883a627b06b6ba2a393d5cf66ddf6ebc4cd0d)) +* bill team-plan service accounts as half a seat ([#11298](https://github.com/windmill-labs/windmill/issues/11298)) ([3aee181](https://github.com/windmill-labs/windmill/commit/3aee1815746f0116f5662f162ad1e234717af90d)) +* card layout and load-more for the audit logs page ([#11507](https://github.com/windmill-labs/windmill/issues/11507)) ([af1a29a](https://github.com/windmill-labs/windmill/commit/af1a29a3cb33386f61a759ec0dd5eec0ccefefc2)) +* offer audit-log and drive-metadata scopes for google oauth connects ([#11527](https://github.com/windmill-labs/windmill/issues/11527)) ([c4e7c71](https://github.com/windmill-labs/windmill/commit/c4e7c714e455f958d9a927c4bc4eb23256324780)) +* offer slack read scopes for files, user groups, emoji and dms ([#11526](https://github.com/windmill-labs/windmill/issues/11526)) ([b5fda7d](https://github.com/windmill-labs/windmill/commit/b5fda7d5af7105fde69beb68410984350af2ac7e)) + + +### Bug Fixes + +* bound s3 ai attachment downloads before encoding ([#11534](https://github.com/windmill-labs/windmill/issues/11534)) ([92fbfc0](https://github.com/windmill-labs/windmill/commit/92fbfc03d219544c16e8f7db47a547f9ff402c27)) +* collapse queued and running chat tool calls into a waiting row ([#11523](https://github.com/windmill-labs/windmill/issues/11523)) ([4f30982](https://github.com/windmill-labs/windmill/commit/4f30982ce3f7abc64a31890f8e88c31c1c2e0803)) +* disable postgres triggers whose replication slot was invalidated ([#11536](https://github.com/windmill-labs/windmill/issues/11536)) ([85248ee](https://github.com/windmill-labs/windmill/commit/85248eee786b633def40c42b6007fc7b907fb09e)) +* evaluate field conditions and step test inputs in QuickJS ([#11487](https://github.com/windmill-labs/windmill/issues/11487)) ([c962d37](https://github.com/windmill-labs/windmill/commit/c962d3766b1a250ba960e413556890d5919ebc94)) +* flush proxied responses before the dev supervisor closes the client ([#11530](https://github.com/windmill-labs/windmill/issues/11530)) ([cefb195](https://github.com/windmill-labs/windmill/commit/cefb1953764e714d79556e2f8065cfbec78acb86)) +* label the default slack scopes and list them first in the connect dialog ([#11529](https://github.com/windmill-labs/windmill/issues/11529)) ([e76781c](https://github.com/windmill-labs/windmill/commit/e76781cc2709fda08403e14c1a0a01957e430db1)) +* run each ai eval model in its own job and let the docs tools reach the backend ([#11533](https://github.com/windmill-labs/windmill/issues/11533)) ([222dad1](https://github.com/windmill-labs/windmill/commit/222dad1d8cc9cbb6925fdff709e00a49bb63bc4a)) +* show the flow row edit button to builder operators ([#11535](https://github.com/windmill-labs/windmill/issues/11535)) ([6c4e34c](https://github.com/windmill-labs/windmill/commit/6c4e34cd0dd9b4329084214c40695f5915947cc2)) + +## [1.823.1](https://github.com/windmill-labs/windmill/compare/v1.823.0...v1.823.1) (2026-10-05) + + +### Bug Fixes + +* python results with a KeyError-raising __getattr__ fail the job ([#11517](https://github.com/windmill-labs/windmill/issues/11517)) ([3fff1bd](https://github.com/windmill-labs/windmill/commit/3fff1bd3dcedce22c58d988ad2e725e3d4e96e0e)) +* stream the ai chat compaction request ([#11514](https://github.com/windmill-labs/windmill/issues/11514)) ([289d3c3](https://github.com/windmill-labs/windmill/commit/289d3c3941c2d9997bf0476bf9bfdfd948542c55)) + +## [1.823.0](https://github.com/windmill-labs/windmill/compare/v1.822.0...v1.823.0) (2026-10-04) + + +### Features + +* redesign the service logs page ([#11506](https://github.com/windmill-labs/windmill/issues/11506)) ([0f65df2](https://github.com/windmill-labs/windmill/commit/0f65df24d2cabd9d8daa71c103969e6392e37b4f)) + + +### Bug Fixes + +* apply the pg keepalive settings to the postgres trigger connection ([#11512](https://github.com/windmill-labs/windmill/issues/11512)) ([33f6de0](https://github.com/windmill-labs/windmill/commit/33f6de0cd20f45ff56770f946a4731e5b13bf677)) +* bound the datatable migration lock wait and reap a dead holder ([#11510](https://github.com/windmill-labs/windmill/issues/11510)) ([711e08c](https://github.com/windmill-labs/windmill/commit/711e08c1d20a326142235c83d8805b7990bf6e6d)) +* bound the pg cached-connection reset probe so a vanished server fails fast ([#11505](https://github.com/windmill-labs/windmill/issues/11505)) ([d4a423f](https://github.com/windmill-labs/windmill/commit/d4a423f567e1fb5e21016a2be61d858ab61a4d4f)) +* cancel an abandoned pg query and close failed connections cleanly ([#11511](https://github.com/windmill-labs/windmill/issues/11511)) ([26856b8](https://github.com/windmill-labs/windmill/commit/26856b8061c28eade191c7d85a32b74ca44c4fea)) +* delete a flow chat turn with its flow run, not its step jobs ([#11477](https://github.com/windmill-labs/windmill/issues/11477)) ([35ec16d](https://github.com/windmill-labs/windmill/commit/35ec16d12728531ab90632116913e98f80d1cc05)) +* drop a fork's datatable database despite connected sessions ([#11509](https://github.com/windmill-labs/windmill/issues/11509)) ([fa419af](https://github.com/windmill-labs/windmill/commit/fa419af9b3f7d65a607bb8ea2bcea58223f743c4)) + +## [1.822.0](https://github.com/windmill-labs/windmill/compare/v1.821.0...v1.822.0) (2026-10-03) + + +### Features + +* add a cancel-only jobs:cancel token scope, optionally path-scoped ([#11479](https://github.com/windmill-labs/windmill/issues/11479)) ([4bc7e0d](https://github.com/windmill-labs/windmill/commit/4bc7e0d7bad22e9e1a3532ca747ed863f90213ba)) +* add an instance setting routing all dependency jobs to one tag ([#11486](https://github.com/windmill-labs/windmill/issues/11486)) ([3d1d249](https://github.com/windmill-labs/windmill/commit/3d1d2495567274b4c48babb374e7716c4ed24ecc)) +* add trigger, tag, run-as and digest claims to job OIDC tokens ([#11481](https://github.com/windmill-labs/windmill/issues/11481)) ([4beb1f9](https://github.com/windmill-labs/windmill/commit/4beb1f9420524425c354e82729e4e35c5bf65451)) +* log pg job progress, warn on stalls, detect dead db connections ([#11491](https://github.com/windmill-labs/windmill/issues/11491)) ([d722772](https://github.com/windmill-labs/windmill/commit/d7227726be2b082bbfb204abb75a3ff081db77de)) +* read the OIDC signing key from a file and rotate it ([#11482](https://github.com/windmill-labs/windmill/issues/11482)) ([d949484](https://github.com/windmill-labs/windmill/commit/d949484e7ba9911cf128387e096f78c917fb3f35)) +* replace the AI input filling toggle with an additional prompt ([#11501](https://github.com/windmill-labs/windmill/issues/11501)) ([e952298](https://github.com/windmill-labs/windmill/commit/e952298f84d04cbb11f0c44c32435f0b711bf3bf)) +* restricted job tokens per script and flow ([#11484](https://github.com/windmill-labs/windmill/issues/11484)) ([e7fc1b2](https://github.com/windmill-labs/windmill/commit/e7fc1b2e2e9b358b414d0ca0eaf2753e142dded7)) + + +### Bug Fixes + +* keep a resource default set in the schema when the script is redeployed ([#11495](https://github.com/windmill-labs/windmill/issues/11495)) ([68f8f19](https://github.com/windmill-labs/windmill/commit/68f8f19b33768451ebab267b36f931a6f386c678)) +* let instances withhold signing secrets from the settings API ([#11483](https://github.com/windmill-labs/windmill/issues/11483)) ([99e96d3](https://github.com/windmill-labs/windmill/commit/99e96d3f781decda8fed2cbd5409cbc5c3520eaa)) +* let legacy draft-only items be discarded from the home page ([#11488](https://github.com/windmill-labs/windmill/issues/11488)) ([98274a7](https://github.com/windmill-labs/windmill/commit/98274a73361618ff37a929584e5b5f519b71a912)) +* never reuse a pg connection a script left inside a transaction ([#11497](https://github.com/windmill-labs/windmill/issues/11497)) ([b7f74cd](https://github.com/windmill-labs/windmill/commit/b7f74cd56294cdb7052e311c5a68de9ee0bb4122)) +* read inet and cidr results with their prefix, including arrays ([#11476](https://github.com/windmill-labs/windmill/issues/11476)) ([ae093b7](https://github.com/windmill-labs/windmill/commit/ae093b7666cefc3777be0e45144bb272732846d0)) +* stop the pg executor cache from pinning a pooler slot ([#11490](https://github.com/windmill-labs/windmill/issues/11490)) ([eed7e7d](https://github.com/windmill-labs/windmill/commit/eed7e7d9e6bb375daebbb323b1288c36a278f977)) +* unstick postgres jobs on large results with custom-typed columns ([#11475](https://github.com/windmill-labs/windmill/issues/11475)) ([14801fd](https://github.com/windmill-labs/windmill/commit/14801fdd68407add824bfa2c679c0e8cb4788610)) + + +### Performance Improvements + +* keep a pg executor connection per database ([#11492](https://github.com/windmill-labs/windmill/issues/11492)) ([013b302](https://github.com/windmill-labs/windmill/commit/013b302d484b967443cbf747fb9f2ecbb1145033)) + ## [1.821.0](https://github.com/windmill-labs/windmill/compare/v1.820.0...v1.821.0) (2026-10-01) diff --git a/Dockerfile b/Dockerfile index 8d229dba56..b1b3f939e7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -142,10 +142,10 @@ FROM ${DEBIAN_IMAGE} ARG TARGETPLATFORM ARG POWERSHELL_VERSION=7.5.0 -ARG KUBECTL_VERSION=1.36.2 +ARG KUBECTL_VERSION=1.37.1 ARG HELM_VERSION=3.21.4 # NOTE: If changing, also change go version in workspace dependencies template at WorkspaceDependenciesEditor.svelte -ARG GO_VERSION=1.26.8 +ARG GO_VERSION=1.27.1 ARG APP=/usr/src/app ARG WITH_POWERSHELL=true ARG WITH_KUBECTL=true diff --git a/ai_evals/adapters/frontend/vitestAdapter.test.ts b/ai_evals/adapters/frontend/vitestAdapter.test.ts index 9adb4d05ac..288c67e3eb 100644 --- a/ai_evals/adapters/frontend/vitestAdapter.test.ts +++ b/ai_evals/adapters/frontend/vitestAdapter.test.ts @@ -6,6 +6,8 @@ import { dirname, resolve } from 'path' // @ts-ignore - Node.js url import { fileURLToPath } from 'url' import { handleBenchmarkApiFetch, hasBenchmarkApiHandler } from './mockBackend' +import { resolveWindmillBackendSettings } from '../../core/windmillBackendSettings' +import { WindmillBackendClient } from './windmillBackend' const FRONTEND_DIR = fileURLToPath(new URL('../../../frontend/', import.meta.url)) @@ -28,6 +30,11 @@ globalThis.fetch = (async (input: unknown, init?: RequestInit) => { if (typeof url === 'string' && hasBenchmarkApiHandler(url)) { return handleBenchmarkApiFetch(url, init) } + // The docs tools read the documentation bundled into the backend: hand them to the + // benchmark's real backend, logged in like the AI proxy calls. + if (typeof url === 'string' && url.startsWith('/api/docs/')) { + return new WindmillBackendClient(resolveWindmillBackendSettings()).request(url.slice('/api'.length), init) + } // The parsers behind inferArgs load their wasm from a vite `?url` path, which only a dev // server serves. Unserved, every script schema infers as empty, and the tools then tell the // model its arguments are undeclared, so it rewrites a correct script until it runs out of turns. @@ -684,6 +691,7 @@ benchmarkIt( resetBenchmarkMockBackend() } }, - // Full-suite runs (30+ cases at concurrency 2-3) routinely exceed 10 minutes. - 7_200_000 + // A full suite on a slow reasoning model runs for hours (gpt-6-astra on global went past + // 2h); the CI job's own timeout is the real bound. + 18_000_000 ) diff --git a/backend/.sqlx/query-05b610ad1809d487c32e85b0392cfca8913ba1129fdf4978a8a4738689786160.json b/backend/.sqlx/query-05b610ad1809d487c32e85b0392cfca8913ba1129fdf4978a8a4738689786160.json new file mode 100644 index 0000000000..5c6ee3d506 --- /dev/null +++ b/backend/.sqlx/query-05b610ad1809d487c32e85b0392cfca8913ba1129fdf4978a8a4738689786160.json @@ -0,0 +1,19 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO script\n (workspace_id, hash, path, parent_hashes, summary, description, content, created_by, schema, is_template, extra_perms, lock, language, kind, tag, envs, concurrent_limit, concurrency_time_window_s, cache_ttl, cache_ignore_s3_path, dedicated_worker, ws_error_handler_muted, priority, restart_unless_cancelled, delete_after_use, delete_after_secs, timeout, concurrency_key, visible_to_runner_only, auto_kind, codebase, has_preprocessor, on_behalf_of, on_behalf_of_email, schema_validation, assets, debounce_key, debounce_delay_s, runnable_settings_handle, modules, labels, job_token_scopes, lock_error_logs, created_at)\n\n SELECT workspace_id, $1, path, array_prepend($2::bigint, COALESCE(parent_hashes, '{}'::bigint[])), summary, description, content, created_by, schema, is_template, extra_perms, $4::text, language, kind, tag, envs, concurrent_limit, concurrency_time_window_s, cache_ttl, cache_ignore_s3_path, dedicated_worker, ws_error_handler_muted, priority, restart_unless_cancelled, delete_after_use, delete_after_secs, timeout, concurrency_key, visible_to_runner_only, auto_kind, codebase, has_preprocessor, on_behalf_of, on_behalf_of_email, schema_validation, assets, debounce_key, debounce_delay_s, runnable_settings_handle, COALESCE($5::jsonb, modules), labels, job_token_scopes, $6::text, clock_timestamp()\n\n FROM script WHERE hash = $2 AND workspace_id = $3;\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int8", + "Int8", + "Text", + "Text", + "Jsonb", + "Text" + ] + }, + "nullable": [] + }, + "hash": "05b610ad1809d487c32e85b0392cfca8913ba1129fdf4978a8a4738689786160" +} diff --git a/backend/.sqlx/query-095fa75c60724664f0355cd6e3a64f84a0f1a31cd6732c0ab994a2d6ac3c3eec.json b/backend/.sqlx/query-095fa75c60724664f0355cd6e3a64f84a0f1a31cd6732c0ab994a2d6ac3c3eec.json new file mode 100644 index 0000000000..852ce508ff --- /dev/null +++ b/backend/.sqlx/query-095fa75c60724664f0355cd6e3a64f84a0f1a31cd6732c0ab994a2d6ac3c3eec.json @@ -0,0 +1,152 @@ +{ + "db_name": "PostgreSQL", + "query": "WITH inserted_job AS (\n INSERT INTO v2_job (\n id, -- 1\n workspace_id, -- 2\n raw_code, -- 3\n raw_lock, -- 4\n raw_flow, -- 5\n tag, -- 6\n parent_job, -- 7\n created_by, -- 8\n permissioned_as, -- 9\n runnable_id, -- 10\n runnable_path, -- 11\n args, -- 12\n kind, -- 13\n trigger, -- 14\n script_lang, -- 15\n same_worker, -- 16\n pre_run_error, -- 17\n permissioned_as_email, -- 18\n visible_to_owner, -- 19\n flow_innermost_root_job, -- 20\n root_job, -- 38\n concurrent_limit, -- 21\n concurrency_time_window_s, -- 22\n timeout, -- 23\n flow_step_id, -- 24\n cache_ttl, -- 25\n priority, -- 26\n trigger_kind, -- 39\n script_entrypoint_override, -- 12\n preprocessed, -- 27,\n labels -- 44\n ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18,\n $19, $20, $38, $21, $22, $23, $24, $25, $26, $39::job_trigger_kind,\n ($12::JSONB)->>'_ENTRYPOINT_OVERRIDE', $27,\n -- $44 (payload labels) merged with the labels of the runnable's folder, if any\n -- ($45/$46 are runnable_path/workspace_id again, kept separate to avoid parameter type conflicts)\n (SELECT CASE WHEN fl.labels IS NULL THEN $44\n ELSE (SELECT array_agg(DISTINCT lbl) FROM unnest(COALESCE($44, ARRAY[]::TEXT[]) || fl.labels) lbl)\n END\n FROM folder_labels($46, $45) AS fl(labels)))\n ),\n inserted_runtime AS (\n INSERT INTO v2_job_runtime (id, ping) VALUES ($1, null)\n ),\n inserted_job_perms AS (\n INSERT INTO job_perms (job_id, email, username, is_admin, is_operator, folders, groups, workspace_id, end_user_email, job_token_scopes)\n values ($1, $32, $33, $34, $35, $36, $37, $2, $41, $47)\n ON CONFLICT (job_id) DO UPDATE SET email = EXCLUDED.email, username = EXCLUDED.username, is_admin = EXCLUDED.is_admin, is_operator = EXCLUDED.is_operator, folders = EXCLUDED.folders, groups = EXCLUDED.groups, workspace_id = EXCLUDED.workspace_id, end_user_email = EXCLUDED.end_user_email, job_token_scopes = EXCLUDED.job_token_scopes\n )\n INSERT INTO v2_job_queue\n (workspace_id, id, running, scheduled_for, started_at, tag, priority, cache_ignore_s3_path, runnable_settings_handle)\n VALUES ($2, $1, $28, COALESCE($29, now()), CASE WHEN $27 OR $40 THEN now() END, $30, $31, $42, $43)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Uuid", + "Varchar", + "Text", + "Text", + "Jsonb", + "Varchar", + "Uuid", + "Varchar", + "Varchar", + "Int8", + "Varchar", + "Jsonb", + { + "Custom": { + "name": "job_kind", + "kind": { + "Enum": [ + "script", + "preview", + "flow", + "dependencies", + "flowpreview", + "script_hub", + "identity", + "flowdependencies", + "http", + "graphql", + "postgresql", + "noop", + "appdependencies", + "deploymentcallback", + "singlestepflow", + "flowscript", + "flownode", + "appscript", + "aiagent", + "unassigned_script", + "unassigned_flow", + "unassigned_singlestepflow" + ] + } + } + }, + "Varchar", + { + "Custom": { + "name": "script_lang", + "kind": { + "Enum": [ + "python3", + "deno", + "go", + "bash", + "postgresql", + "nativets", + "bun", + "mysql", + "bigquery", + "snowflake", + "graphql", + "powershell", + "mssql", + "php", + "bunnative", + "rust", + "ansible", + "csharp", + "oracledb", + "nu", + "java", + "duckdb", + "ruby", + "rlang", + "dbt" + ] + } + } + }, + "Bool", + "Text", + "Varchar", + "Bool", + "Uuid", + "Int4", + "Int4", + "Int4", + "Varchar", + "Int4", + "Int2", + "Bool", + "Bool", + "Timestamptz", + "Varchar", + "Int2", + "Varchar", + "Varchar", + "Bool", + "Bool", + "JsonbArray", + "TextArray", + "Uuid", + { + "Custom": { + "name": "job_trigger_kind", + "kind": { + "Enum": [ + "webhook", + "http", + "websocket", + "kafka", + "email", + "nats", + "schedule", + "app", + "ui", + "postgres", + "sqs", + "gcp", + "mqtt", + "nextcloud", + "google", + "ci_test", + "github", + "azure", + "asset", + "freshness", + "amqp" + ] + } + } + }, + "Bool", + "Varchar", + "Bool", + "Int8", + "TextArray", + "Text", + "Text", + "TextArray" + ] + }, + "nullable": [] + }, + "hash": "095fa75c60724664f0355cd6e3a64f84a0f1a31cd6732c0ab994a2d6ac3c3eec" +} diff --git a/backend/.sqlx/query-0d260ba4f168d725ea3b82d33324dd1d16a68a2bae6e500d9e480b152fd5859d.json b/backend/.sqlx/query-0d260ba4f168d725ea3b82d33324dd1d16a68a2bae6e500d9e480b152fd5859d.json new file mode 100644 index 0000000000..28dfcf702f --- /dev/null +++ b/backend/.sqlx/query-0d260ba4f168d725ea3b82d33324dd1d16a68a2bae6e500d9e480b152fd5859d.json @@ -0,0 +1,47 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT DISTINCT ON (path)\n path,\n value as \"value!: sqlx::types::Json>\",\n created_at,\n email IS NULL as \"legacy!\",\n typ::text as \"typ!\"\n FROM draft\n WHERE workspace_id = $1\n AND typ IN ('app', 'raw_app')\n AND (email = $2 OR email IS NULL)\n AND NOT EXISTS (\n SELECT 1 FROM app a\n WHERE a.workspace_id = draft.workspace_id\n AND a.path = draft.path\n )\n ORDER BY path, (email IS NULL), created_at DESC", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "path", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "value!: sqlx::types::Json>", + "type_info": "Json" + }, + { + "ordinal": 2, + "name": "created_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 3, + "name": "legacy!", + "type_info": "Bool" + }, + { + "ordinal": 4, + "name": "typ!", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false, + false, + false, + null, + null + ] + }, + "hash": "0d260ba4f168d725ea3b82d33324dd1d16a68a2bae6e500d9e480b152fd5859d" +} diff --git a/backend/.sqlx/query-10f166464a10fc35df2b6ff1b497b17d6bb71f0ae69cd218e6a3fbc151d19d87.json b/backend/.sqlx/query-10f166464a10fc35df2b6ff1b497b17d6bb71f0ae69cd218e6a3fbc151d19d87.json new file mode 100644 index 0000000000..027f01d181 --- /dev/null +++ b/backend/.sqlx/query-10f166464a10fc35df2b6ff1b497b17d6bb71f0ae69cd218e6a3fbc151d19d87.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT EXISTS (\n SELECT 1 FROM v2_job o, v2_job t,\n LATERAL (SELECT ARRAY_REMOVE(ARRAY[o.id, o.parent_job, o.root_job,\n o.flow_innermost_root_job], NULL) AS run) l\n WHERE o.id = $1 AND t.id = $2 AND t.workspace_id = o.workspace_id\n AND (t.id = ANY(l.run) OR t.parent_job = ANY(l.run)\n OR t.root_job = ANY(l.run) OR t.flow_innermost_root_job = ANY(l.run))\n )", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "exists", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Uuid" + ] + }, + "nullable": [ + null + ] + }, + "hash": "10f166464a10fc35df2b6ff1b497b17d6bb71f0ae69cd218e6a3fbc151d19d87" +} diff --git a/backend/.sqlx/query-13f358b7c76cda0b24fcb17dbbe10a00333bbf3b8b43bbccc7a6e1926d82478c.json b/backend/.sqlx/query-13f358b7c76cda0b24fcb17dbbe10a00333bbf3b8b43bbccc7a6e1926d82478c.json new file mode 100644 index 0000000000..25aedfe3be --- /dev/null +++ b/backend/.sqlx/query-13f358b7c76cda0b24fcb17dbbe10a00333bbf3b8b43bbccc7a6e1926d82478c.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT job_token_scopes FROM job_perms WHERE job_id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "job_token_scopes", + "type_info": "TextArray" + } + ], + "parameters": { + "Left": [ + "Uuid" + ] + }, + "nullable": [ + true + ] + }, + "hash": "13f358b7c76cda0b24fcb17dbbe10a00333bbf3b8b43bbccc7a6e1926d82478c" +} diff --git a/backend/.sqlx/query-333f7116d660756c36dfc0aa2b8d7ebd10a38cbe5724a554a12291c287b67f6d.json b/backend/.sqlx/query-333f7116d660756c36dfc0aa2b8d7ebd10a38cbe5724a554a12291c287b67f6d.json new file mode 100644 index 0000000000..23b46f5577 --- /dev/null +++ b/backend/.sqlx/query-333f7116d660756c36dfc0aa2b8d7ebd10a38cbe5724a554a12291c287b67f6d.json @@ -0,0 +1,324 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT\n v2_job_queue.workspace_id,\n v2_job_queue.id,\n v2_job.args as \"args: sqlx::types::Json>>\",\n v2_job.parent_job,\n v2_job.created_by,\n v2_job_queue.started_at,\n v2_job_queue.runnable_settings_handle,\n scheduled_for,\n runnable_path,\n kind as \"kind: JobKind\",\n runnable_id as \"runnable_id: ScriptHash\",\n canceled_reason,\n canceled_by,\n permissioned_as,\n permissioned_as_email,\n flow_status as \"flow_status: sqlx::types::Json>\",\n v2_job.tag,\n script_lang as \"script_lang: ScriptLang\",\n same_worker,\n pre_run_error,\n concurrent_limit,\n concurrency_time_window_s,\n flow_innermost_root_job,\n root_job,\n timeout,\n flow_step_id,\n cache_ttl,\n cache_ignore_s3_path,\n v2_job_queue.priority,\n preprocessed,\n script_entrypoint_override,\n trigger,\n trigger_kind as \"trigger_kind: TriggerKindLabel\",\n visible_to_owner,\n NULL as permissioned_as_end_user_email,\n job_perms.job_token_scopes\n FROM v2_job_queue INNER JOIN v2_job ON v2_job.id = v2_job_queue.id LEFT JOIN v2_job_status ON v2_job_status.id = v2_job_queue.id\n LEFT JOIN job_perms ON job_perms.job_id = v2_job_queue.id WHERE v2_job_queue.id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "workspace_id", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "id", + "type_info": "Uuid" + }, + { + "ordinal": 2, + "name": "args: sqlx::types::Json>>", + "type_info": "Jsonb" + }, + { + "ordinal": 3, + "name": "parent_job", + "type_info": "Uuid" + }, + { + "ordinal": 4, + "name": "created_by", + "type_info": "Varchar" + }, + { + "ordinal": 5, + "name": "started_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 6, + "name": "runnable_settings_handle", + "type_info": "Int8" + }, + { + "ordinal": 7, + "name": "scheduled_for", + "type_info": "Timestamptz" + }, + { + "ordinal": 8, + "name": "runnable_path", + "type_info": "Varchar" + }, + { + "ordinal": 9, + "name": "kind: JobKind", + "type_info": { + "Custom": { + "name": "job_kind", + "kind": { + "Enum": [ + "script", + "preview", + "flow", + "dependencies", + "flowpreview", + "script_hub", + "identity", + "flowdependencies", + "http", + "graphql", + "postgresql", + "noop", + "appdependencies", + "deploymentcallback", + "singlestepflow", + "flowscript", + "flownode", + "appscript", + "aiagent", + "unassigned_script", + "unassigned_flow", + "unassigned_singlestepflow" + ] + } + } + } + }, + { + "ordinal": 10, + "name": "runnable_id: ScriptHash", + "type_info": "Int8" + }, + { + "ordinal": 11, + "name": "canceled_reason", + "type_info": "Text" + }, + { + "ordinal": 12, + "name": "canceled_by", + "type_info": "Varchar" + }, + { + "ordinal": 13, + "name": "permissioned_as", + "type_info": "Varchar" + }, + { + "ordinal": 14, + "name": "permissioned_as_email", + "type_info": "Varchar" + }, + { + "ordinal": 15, + "name": "flow_status: sqlx::types::Json>", + "type_info": "Jsonb" + }, + { + "ordinal": 16, + "name": "tag", + "type_info": "Varchar" + }, + { + "ordinal": 17, + "name": "script_lang: ScriptLang", + "type_info": { + "Custom": { + "name": "script_lang", + "kind": { + "Enum": [ + "python3", + "deno", + "go", + "bash", + "postgresql", + "nativets", + "bun", + "mysql", + "bigquery", + "snowflake", + "graphql", + "powershell", + "mssql", + "php", + "bunnative", + "rust", + "ansible", + "csharp", + "oracledb", + "nu", + "java", + "duckdb", + "ruby", + "rlang", + "dbt" + ] + } + } + } + }, + { + "ordinal": 18, + "name": "same_worker", + "type_info": "Bool" + }, + { + "ordinal": 19, + "name": "pre_run_error", + "type_info": "Text" + }, + { + "ordinal": 20, + "name": "concurrent_limit", + "type_info": "Int4" + }, + { + "ordinal": 21, + "name": "concurrency_time_window_s", + "type_info": "Int4" + }, + { + "ordinal": 22, + "name": "flow_innermost_root_job", + "type_info": "Uuid" + }, + { + "ordinal": 23, + "name": "root_job", + "type_info": "Uuid" + }, + { + "ordinal": 24, + "name": "timeout", + "type_info": "Int4" + }, + { + "ordinal": 25, + "name": "flow_step_id", + "type_info": "Varchar" + }, + { + "ordinal": 26, + "name": "cache_ttl", + "type_info": "Int4" + }, + { + "ordinal": 27, + "name": "cache_ignore_s3_path", + "type_info": "Bool" + }, + { + "ordinal": 28, + "name": "priority", + "type_info": "Int2" + }, + { + "ordinal": 29, + "name": "preprocessed", + "type_info": "Bool" + }, + { + "ordinal": 30, + "name": "script_entrypoint_override", + "type_info": "Varchar" + }, + { + "ordinal": 31, + "name": "trigger", + "type_info": "Varchar" + }, + { + "ordinal": 32, + "name": "trigger_kind: TriggerKindLabel", + "type_info": { + "Custom": { + "name": "job_trigger_kind", + "kind": { + "Enum": [ + "webhook", + "http", + "websocket", + "kafka", + "email", + "nats", + "schedule", + "app", + "ui", + "postgres", + "sqs", + "gcp", + "mqtt", + "nextcloud", + "google", + "ci_test", + "github", + "azure", + "asset", + "freshness", + "amqp" + ] + } + } + } + }, + { + "ordinal": 33, + "name": "visible_to_owner", + "type_info": "Bool" + }, + { + "ordinal": 34, + "name": "permissioned_as_end_user_email", + "type_info": "Text" + }, + { + "ordinal": 35, + "name": "job_token_scopes", + "type_info": "TextArray" + } + ], + "parameters": { + "Left": [ + "Uuid" + ] + }, + "nullable": [ + false, + false, + true, + true, + false, + true, + true, + false, + true, + false, + true, + true, + true, + false, + false, + true, + false, + true, + false, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + false, + null, + true + ] + }, + "hash": "333f7116d660756c36dfc0aa2b8d7ebd10a38cbe5724a554a12291c287b67f6d" +} diff --git a/backend/.sqlx/query-35f960e9de880fdcaf81615933c828c4a877709c3c39a530d629cc91f9dd2d29.json b/backend/.sqlx/query-35f960e9de880fdcaf81615933c828c4a877709c3c39a530d629cc91f9dd2d29.json new file mode 100644 index 0000000000..db2af83b47 --- /dev/null +++ b/backend/.sqlx/query-35f960e9de880fdcaf81615933c828c4a877709c3c39a530d629cc91f9dd2d29.json @@ -0,0 +1,29 @@ +{ + "db_name": "PostgreSQL", + "query": "WITH RECURSIVE chain(origin, id, parent_job) AS (\n SELECT id, id, parent_job FROM v2_job WHERE id = ANY($1) AND workspace_id = $2\n UNION ALL\n SELECT c.origin, j.id, j.parent_job FROM v2_job j\n JOIN chain c ON j.id = c.parent_job AND j.workspace_id = $2\n )\n SELECT c.origin AS \"origin!\",\n CASE WHEN a.agent THEN regexp_replace(j.runnable_path, '\\.chat$', '')\n ELSE j.runnable_path END AS \"runnable_path!\"\n FROM chain c JOIN v2_job j ON j.id = c.id,\n LATERAL (SELECT j.kind = 'flowpreview'\n AND j.raw_flow->'modules'->1 IS NULL\n AND j.raw_flow->'modules'->0->>'id' = '__wm_agent_root' AS agent) a\n WHERE j.workspace_id = $2 AND j.runnable_path IS NOT NULL\n AND (a.agent OR j.kind IN ('script', 'script_hub', 'unassigned_script', 'flow',\n 'unassigned_flow', 'singlestepflow', 'unassigned_singlestepflow'))", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "origin!", + "type_info": "Uuid" + }, + { + "ordinal": 1, + "name": "runnable_path!", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "UuidArray", + "Text" + ] + }, + "nullable": [ + null, + null + ] + }, + "hash": "35f960e9de880fdcaf81615933c828c4a877709c3c39a530d629cc91f9dd2d29" +} diff --git a/backend/.sqlx/query-4631200a37f7edec92f3796ab4d4585d9c2aba36fdf33bca9739049366bed107.json b/backend/.sqlx/query-4631200a37f7edec92f3796ab4d4585d9c2aba36fdf33bca9739049366bed107.json new file mode 100644 index 0000000000..a5bde4d661 --- /dev/null +++ b/backend/.sqlx/query-4631200a37f7edec92f3796ab4d4585d9c2aba36fdf33bca9739049366bed107.json @@ -0,0 +1,84 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT\n flow_version.id AS version,\n flow_version.value->>'early_return' as early_return,\n flow_version.value->>'preprocessor_module' IS NOT NULL as has_preprocessor,\n flow_version.value->>'failure_module' IS NOT NULL as has_failure_module,\n (flow_version.value->>'chat_input_enabled')::boolean as chat_input_enabled,\n flow.tag,\n flow.dedicated_worker,\n flow.on_behalf_of,\n flow.edited_by,\n flow.labels,\n flow.job_token_scopes\n FROM\n flow_version\n INNER JOIN flow\n ON flow.path = flow_version.path AND\n flow.workspace_id = flow_version.workspace_id\n WHERE\n flow_version.workspace_id = $1 AND\n flow_version.path = $2 AND\n flow_version.id = $3\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "version", + "type_info": "Int8" + }, + { + "ordinal": 1, + "name": "early_return", + "type_info": "Text" + }, + { + "ordinal": 2, + "name": "has_preprocessor", + "type_info": "Bool" + }, + { + "ordinal": 3, + "name": "has_failure_module", + "type_info": "Bool" + }, + { + "ordinal": 4, + "name": "chat_input_enabled", + "type_info": "Bool" + }, + { + "ordinal": 5, + "name": "tag", + "type_info": "Varchar" + }, + { + "ordinal": 6, + "name": "dedicated_worker", + "type_info": "Bool" + }, + { + "ordinal": 7, + "name": "on_behalf_of", + "type_info": "Varchar" + }, + { + "ordinal": 8, + "name": "edited_by", + "type_info": "Varchar" + }, + { + "ordinal": 9, + "name": "labels", + "type_info": "TextArray" + }, + { + "ordinal": 10, + "name": "job_token_scopes", + "type_info": "TextArray" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Int8" + ] + }, + "nullable": [ + false, + null, + null, + null, + null, + true, + true, + true, + false, + true, + true + ] + }, + "hash": "4631200a37f7edec92f3796ab4d4585d9c2aba36fdf33bca9739049366bed107" +} diff --git a/backend/.sqlx/query-4cc81f5e04caefa4f170807c0ce96c69f22d9ea9dc3586099553a8f0e76aad82.json b/backend/.sqlx/query-4cc81f5e04caefa4f170807c0ce96c69f22d9ea9dc3586099553a8f0e76aad82.json new file mode 100644 index 0000000000..7051c33008 --- /dev/null +++ b/backend/.sqlx/query-4cc81f5e04caefa4f170807c0ce96c69f22d9ea9dc3586099553a8f0e76aad82.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO job_perms (job_id, email, username, is_admin, is_operator, folders, groups, workspace_id, job_token_scopes)\n SELECT unnest($1::uuid[]), $2, $3, $4, $5, $6, $7, $8, $9", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "UuidArray", + "Varchar", + "Varchar", + "Bool", + "Bool", + "JsonbArray", + "TextArray", + "Varchar", + "TextArray" + ] + }, + "nullable": [] + }, + "hash": "4cc81f5e04caefa4f170807c0ce96c69f22d9ea9dc3586099553a8f0e76aad82" +} diff --git a/backend/.sqlx/query-5223412e366b22d953867425ed6156a66727e7a469abad1664ee03a3da31078a.json b/backend/.sqlx/query-5223412e366b22d953867425ed6156a66727e7a469abad1664ee03a3da31078a.json deleted file mode 100644 index be5e26ef9b..0000000000 --- a/backend/.sqlx/query-5223412e366b22d953867425ed6156a66727e7a469abad1664ee03a3da31078a.json +++ /dev/null @@ -1,35 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT DISTINCT ON (path)\n path,\n value as \"value!: sqlx::types::Json>\",\n created_at\n FROM draft\n WHERE workspace_id = $1\n AND typ = 'flow'\n AND (email = $2 OR email IS NULL)\n AND NOT EXISTS (\n SELECT 1 FROM flow f\n WHERE f.workspace_id = draft.workspace_id\n AND f.path = draft.path\n )\n ORDER BY path, (email IS NULL)", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "path", - "type_info": "Varchar" - }, - { - "ordinal": 1, - "name": "value!: sqlx::types::Json>", - "type_info": "Json" - }, - { - "ordinal": 2, - "name": "created_at", - "type_info": "Timestamptz" - } - ], - "parameters": { - "Left": [ - "Text", - "Text" - ] - }, - "nullable": [ - false, - false, - false - ] - }, - "hash": "5223412e366b22d953867425ed6156a66727e7a469abad1664ee03a3da31078a" -} diff --git a/backend/.sqlx/query-580271c5fe178f5e31f3e211cd7fea97aec4d6d3ec65d3e7402811c7dddbbf64.json b/backend/.sqlx/query-580271c5fe178f5e31f3e211cd7fea97aec4d6d3ec65d3e7402811c7dddbbf64.json new file mode 100644 index 0000000000..e1f4609346 --- /dev/null +++ b/backend/.sqlx/query-580271c5fe178f5e31f3e211cd7fea97aec4d6d3ec65d3e7402811c7dddbbf64.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO flow\n (workspace_id, path, summary, description, archived, extra_perms, dependency_job, tag, ws_error_handler_muted, dedicated_worker, timeout, visible_to_runner_only, on_behalf_of, on_behalf_of_email, concurrency_key, versions, value, schema, edited_by, edited_at, labels, job_token_scopes)\n SELECT workspace_id, $1, summary, description, archived, extra_perms, dependency_job, tag, ws_error_handler_muted, dedicated_worker, timeout, visible_to_runner_only, on_behalf_of, on_behalf_of_email, concurrency_key, versions, value, schema, edited_by, edited_at, labels, job_token_scopes\n FROM flow\n WHERE path = $2 AND workspace_id = $3", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "580271c5fe178f5e31f3e211cd7fea97aec4d6d3ec65d3e7402811c7dddbbf64" +} diff --git a/backend/.sqlx/query-5a975e486a0cdb0fae4dadd344ad3abc2f67827597ef74fef473e309b1951544.json b/backend/.sqlx/query-5a975e486a0cdb0fae4dadd344ad3abc2f67827597ef74fef473e309b1951544.json new file mode 100644 index 0000000000..56f5ddbc84 --- /dev/null +++ b/backend/.sqlx/query-5a975e486a0cdb0fae4dadd344ad3abc2f67827597ef74fef473e309b1951544.json @@ -0,0 +1,253 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT\n j.id, j.workspace_id, j.runnable_id AS \"runnable_id: ScriptHash\", q.scheduled_for, q.started_at, j.parent_job, j.flow_innermost_root_job, j.runnable_path, j.kind as \"kind!: JobKind\", j.permissioned_as,\n j.created_by, j.script_lang AS \"script_lang: ScriptLang\", j.permissioned_as_email, j.flow_step_id, j.trigger_kind AS \"trigger_kind: TriggerKindLabel\", j.trigger, j.priority, j.concurrent_limit, j.tag, j.cache_ttl, q.cache_ignore_s3_path, q.runnable_settings_handle,\n COALESCE(j.args->'build_binary_only' = 'true'::jsonb, false) AS \"build_binary_only!\",\n p.job_token_scopes AS \"job_token_scopes?\"\n FROM v2_job j LEFT JOIN v2_job_queue q ON j.id = q.id\n LEFT JOIN job_perms p ON p.job_id = j.id\n WHERE j.id = $1 AND j.workspace_id = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Uuid" + }, + { + "ordinal": 1, + "name": "workspace_id", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "runnable_id: ScriptHash", + "type_info": "Int8" + }, + { + "ordinal": 3, + "name": "scheduled_for", + "type_info": "Timestamptz" + }, + { + "ordinal": 4, + "name": "started_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 5, + "name": "parent_job", + "type_info": "Uuid" + }, + { + "ordinal": 6, + "name": "flow_innermost_root_job", + "type_info": "Uuid" + }, + { + "ordinal": 7, + "name": "runnable_path", + "type_info": "Varchar" + }, + { + "ordinal": 8, + "name": "kind!: JobKind", + "type_info": { + "Custom": { + "name": "job_kind", + "kind": { + "Enum": [ + "script", + "preview", + "flow", + "dependencies", + "flowpreview", + "script_hub", + "identity", + "flowdependencies", + "http", + "graphql", + "postgresql", + "noop", + "appdependencies", + "deploymentcallback", + "singlestepflow", + "flowscript", + "flownode", + "appscript", + "aiagent", + "unassigned_script", + "unassigned_flow", + "unassigned_singlestepflow" + ] + } + } + } + }, + { + "ordinal": 9, + "name": "permissioned_as", + "type_info": "Varchar" + }, + { + "ordinal": 10, + "name": "created_by", + "type_info": "Varchar" + }, + { + "ordinal": 11, + "name": "script_lang: ScriptLang", + "type_info": { + "Custom": { + "name": "script_lang", + "kind": { + "Enum": [ + "python3", + "deno", + "go", + "bash", + "postgresql", + "nativets", + "bun", + "mysql", + "bigquery", + "snowflake", + "graphql", + "powershell", + "mssql", + "php", + "bunnative", + "rust", + "ansible", + "csharp", + "oracledb", + "nu", + "java", + "duckdb", + "ruby", + "rlang", + "dbt" + ] + } + } + } + }, + { + "ordinal": 12, + "name": "permissioned_as_email", + "type_info": "Varchar" + }, + { + "ordinal": 13, + "name": "flow_step_id", + "type_info": "Varchar" + }, + { + "ordinal": 14, + "name": "trigger_kind: TriggerKindLabel", + "type_info": { + "Custom": { + "name": "job_trigger_kind", + "kind": { + "Enum": [ + "webhook", + "http", + "websocket", + "kafka", + "email", + "nats", + "schedule", + "app", + "ui", + "postgres", + "sqs", + "gcp", + "mqtt", + "nextcloud", + "google", + "ci_test", + "github", + "azure", + "asset", + "freshness", + "amqp" + ] + } + } + } + }, + { + "ordinal": 15, + "name": "trigger", + "type_info": "Varchar" + }, + { + "ordinal": 16, + "name": "priority", + "type_info": "Int2" + }, + { + "ordinal": 17, + "name": "concurrent_limit", + "type_info": "Int4" + }, + { + "ordinal": 18, + "name": "tag", + "type_info": "Varchar" + }, + { + "ordinal": 19, + "name": "cache_ttl", + "type_info": "Int4" + }, + { + "ordinal": 20, + "name": "cache_ignore_s3_path", + "type_info": "Bool" + }, + { + "ordinal": 21, + "name": "runnable_settings_handle", + "type_info": "Int8" + }, + { + "ordinal": 22, + "name": "build_binary_only!", + "type_info": "Bool" + }, + { + "ordinal": 23, + "name": "job_token_scopes?", + "type_info": "TextArray" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Text" + ] + }, + "nullable": [ + false, + false, + true, + false, + true, + true, + true, + true, + false, + false, + false, + true, + false, + true, + true, + true, + true, + true, + false, + true, + true, + true, + null, + true + ] + }, + "hash": "5a975e486a0cdb0fae4dadd344ad3abc2f67827597ef74fef473e309b1951544" +} diff --git a/backend/.sqlx/query-60d53c73e67dd7b29cce910fd76b6c87f2994fa2d5f33e0f3ee9effe28614abe.json b/backend/.sqlx/query-60d53c73e67dd7b29cce910fd76b6c87f2994fa2d5f33e0f3ee9effe28614abe.json new file mode 100644 index 0000000000..f04f994e6a --- /dev/null +++ b/backend/.sqlx/query-60d53c73e67dd7b29cce910fd76b6c87f2994fa2d5f33e0f3ee9effe28614abe.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT job_token_scopes FROM script WHERE hash = $1 AND workspace_id = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "job_token_scopes", + "type_info": "TextArray" + } + ], + "parameters": { + "Left": [ + "Int8", + "Text" + ] + }, + "nullable": [ + true + ] + }, + "hash": "60d53c73e67dd7b29cce910fd76b6c87f2994fa2d5f33e0f3ee9effe28614abe" +} diff --git a/backend/.sqlx/query-6445041033c95fcf52c79c33f34cf6c41fb443f8bd91f12bce0e53f6e91bfb16.json b/backend/.sqlx/query-6445041033c95fcf52c79c33f34cf6c41fb443f8bd91f12bce0e53f6e91bfb16.json new file mode 100644 index 0000000000..abcdf7baab --- /dev/null +++ b/backend/.sqlx/query-6445041033c95fcf52c79c33f34cf6c41fb443f8bd91f12bce0e53f6e91bfb16.json @@ -0,0 +1,331 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE v2_job_status SET\n flow_status = JSONB_SET(\n JSONB_SET(v2_job_status.flow_status, $2::TEXT[], $3),\n $4::TEXT[], $5\n ) - $6::TEXT[],\n flow_leaf_jobs = CASE\n WHEN $7::TEXT IS NULL\n OR COALESCE(v2_job.flow_innermost_root_job, v2_job_status.id) <> v2_job_status.id\n THEN v2_job_status.flow_leaf_jobs\n ELSE JSONB_SET(COALESCE(v2_job_status.flow_leaf_jobs, '{}'::JSONB), ARRAY[$7::TEXT], $8) END\n FROM v2_job_queue INNER JOIN v2_job ON v2_job.id = v2_job_queue.id\n LEFT JOIN job_perms ON job_perms.job_id = v2_job_queue.id\n WHERE v2_job_status.id = $1 AND v2_job_queue.id = $1\n RETURNING\n v2_job_queue.workspace_id,\n v2_job_queue.id,\n v2_job.args as \"args: sqlx::types::Json>>\",\n v2_job.parent_job,\n v2_job.created_by,\n v2_job_queue.started_at,\n v2_job_queue.runnable_settings_handle,\n v2_job_queue.scheduled_for,\n v2_job.runnable_path,\n v2_job.kind as \"kind: JobKind\",\n v2_job.runnable_id as \"runnable_id: ScriptHash\",\n v2_job_queue.canceled_reason,\n v2_job_queue.canceled_by,\n v2_job.permissioned_as,\n v2_job.permissioned_as_email,\n v2_job_status.flow_status as \"flow_status: sqlx::types::Json>\",\n v2_job.tag,\n v2_job.script_lang as \"script_lang: ScriptLang\",\n v2_job.same_worker,\n v2_job.pre_run_error,\n v2_job.concurrent_limit,\n v2_job.concurrency_time_window_s,\n v2_job.flow_innermost_root_job,\n v2_job.root_job,\n v2_job.timeout,\n v2_job.flow_step_id,\n v2_job.cache_ttl,\n v2_job_queue.cache_ignore_s3_path,\n v2_job_queue.priority,\n v2_job.preprocessed,\n v2_job.script_entrypoint_override,\n v2_job.trigger,\n v2_job.trigger_kind as \"trigger_kind: TriggerKindLabel\",\n v2_job.visible_to_owner,\n NULL as permissioned_as_end_user_email,\n job_perms.job_token_scopes", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "workspace_id", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "id", + "type_info": "Uuid" + }, + { + "ordinal": 2, + "name": "args: sqlx::types::Json>>", + "type_info": "Jsonb" + }, + { + "ordinal": 3, + "name": "parent_job", + "type_info": "Uuid" + }, + { + "ordinal": 4, + "name": "created_by", + "type_info": "Varchar" + }, + { + "ordinal": 5, + "name": "started_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 6, + "name": "runnable_settings_handle", + "type_info": "Int8" + }, + { + "ordinal": 7, + "name": "scheduled_for", + "type_info": "Timestamptz" + }, + { + "ordinal": 8, + "name": "runnable_path", + "type_info": "Varchar" + }, + { + "ordinal": 9, + "name": "kind: JobKind", + "type_info": { + "Custom": { + "name": "job_kind", + "kind": { + "Enum": [ + "script", + "preview", + "flow", + "dependencies", + "flowpreview", + "script_hub", + "identity", + "flowdependencies", + "http", + "graphql", + "postgresql", + "noop", + "appdependencies", + "deploymentcallback", + "singlestepflow", + "flowscript", + "flownode", + "appscript", + "aiagent", + "unassigned_script", + "unassigned_flow", + "unassigned_singlestepflow" + ] + } + } + } + }, + { + "ordinal": 10, + "name": "runnable_id: ScriptHash", + "type_info": "Int8" + }, + { + "ordinal": 11, + "name": "canceled_reason", + "type_info": "Text" + }, + { + "ordinal": 12, + "name": "canceled_by", + "type_info": "Varchar" + }, + { + "ordinal": 13, + "name": "permissioned_as", + "type_info": "Varchar" + }, + { + "ordinal": 14, + "name": "permissioned_as_email", + "type_info": "Varchar" + }, + { + "ordinal": 15, + "name": "flow_status: sqlx::types::Json>", + "type_info": "Jsonb" + }, + { + "ordinal": 16, + "name": "tag", + "type_info": "Varchar" + }, + { + "ordinal": 17, + "name": "script_lang: ScriptLang", + "type_info": { + "Custom": { + "name": "script_lang", + "kind": { + "Enum": [ + "python3", + "deno", + "go", + "bash", + "postgresql", + "nativets", + "bun", + "mysql", + "bigquery", + "snowflake", + "graphql", + "powershell", + "mssql", + "php", + "bunnative", + "rust", + "ansible", + "csharp", + "oracledb", + "nu", + "java", + "duckdb", + "ruby", + "rlang", + "dbt" + ] + } + } + } + }, + { + "ordinal": 18, + "name": "same_worker", + "type_info": "Bool" + }, + { + "ordinal": 19, + "name": "pre_run_error", + "type_info": "Text" + }, + { + "ordinal": 20, + "name": "concurrent_limit", + "type_info": "Int4" + }, + { + "ordinal": 21, + "name": "concurrency_time_window_s", + "type_info": "Int4" + }, + { + "ordinal": 22, + "name": "flow_innermost_root_job", + "type_info": "Uuid" + }, + { + "ordinal": 23, + "name": "root_job", + "type_info": "Uuid" + }, + { + "ordinal": 24, + "name": "timeout", + "type_info": "Int4" + }, + { + "ordinal": 25, + "name": "flow_step_id", + "type_info": "Varchar" + }, + { + "ordinal": 26, + "name": "cache_ttl", + "type_info": "Int4" + }, + { + "ordinal": 27, + "name": "cache_ignore_s3_path", + "type_info": "Bool" + }, + { + "ordinal": 28, + "name": "priority", + "type_info": "Int2" + }, + { + "ordinal": 29, + "name": "preprocessed", + "type_info": "Bool" + }, + { + "ordinal": 30, + "name": "script_entrypoint_override", + "type_info": "Varchar" + }, + { + "ordinal": 31, + "name": "trigger", + "type_info": "Varchar" + }, + { + "ordinal": 32, + "name": "trigger_kind: TriggerKindLabel", + "type_info": { + "Custom": { + "name": "job_trigger_kind", + "kind": { + "Enum": [ + "webhook", + "http", + "websocket", + "kafka", + "email", + "nats", + "schedule", + "app", + "ui", + "postgres", + "sqs", + "gcp", + "mqtt", + "nextcloud", + "google", + "ci_test", + "github", + "azure", + "asset", + "freshness", + "amqp" + ] + } + } + } + }, + { + "ordinal": 33, + "name": "visible_to_owner", + "type_info": "Bool" + }, + { + "ordinal": 34, + "name": "permissioned_as_end_user_email", + "type_info": "Text" + }, + { + "ordinal": 35, + "name": "job_token_scopes", + "type_info": "TextArray" + } + ], + "parameters": { + "Left": [ + "Uuid", + "TextArray", + "Jsonb", + "TextArray", + "Jsonb", + "TextArray", + "Text", + "Jsonb" + ] + }, + "nullable": [ + false, + false, + true, + true, + false, + true, + true, + false, + true, + false, + true, + true, + true, + false, + false, + true, + false, + true, + false, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + false, + null, + true + ] + }, + "hash": "6445041033c95fcf52c79c33f34cf6c41fb443f8bd91f12bce0e53f6e91bfb16" +} diff --git a/backend/.sqlx/query-67db536d06c57f195641f178ce651cbd7b8896984115d55318bb6f4100d72d9a.json b/backend/.sqlx/query-67db536d06c57f195641f178ce651cbd7b8896984115d55318bb6f4100d72d9a.json new file mode 100644 index 0000000000..cfd88ed5cb --- /dev/null +++ b/backend/.sqlx/query-67db536d06c57f195641f178ce651cbd7b8896984115d55318bb6f4100d72d9a.json @@ -0,0 +1,24 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT $2 IN (parent_job, root_job, flow_innermost_root_job) FROM v2_job\n WHERE id = $1 AND workspace_id = $3", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "?column?", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Uuid", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "67db536d06c57f195641f178ce651cbd7b8896984115d55318bb6f4100d72d9a" +} diff --git a/backend/.sqlx/query-706f2227e34971d22a577b33f6aaa7f5755da9997419c4a9d07b30d4da318bea.json b/backend/.sqlx/query-706f2227e34971d22a577b33f6aaa7f5755da9997419c4a9d07b30d4da318bea.json new file mode 100644 index 0000000000..7c305d1e7a --- /dev/null +++ b/backend/.sqlx/query-706f2227e34971d22a577b33f6aaa7f5755da9997419c4a9d07b30d4da318bea.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO script (\n workspace_id, hash, path, parent_hashes, summary, description, content,\n created_by, created_at, archived, schema, deleted, is_template,\n extra_perms, lock, lock_error_logs, language, kind, tag,\n envs, concurrent_limit, concurrency_time_window_s, cache_ttl,\n dedicated_worker, ws_error_handler_muted, priority, timeout,\n delete_after_use, delete_after_secs, restart_unless_cancelled, concurrency_key,\n visible_to_runner_only, auto_kind, codebase, has_preprocessor,\n on_behalf_of, on_behalf_of_email, assets, modules, job_token_scopes\n )\n SELECT\n $1, hash, path, parent_hashes, summary, description, content,\n created_by, created_at, archived, schema, deleted, is_template,\n extra_perms, lock, lock_error_logs, language, kind, tag,\n envs, concurrent_limit, concurrency_time_window_s, cache_ttl,\n dedicated_worker, ws_error_handler_muted, priority, timeout,\n delete_after_use, delete_after_secs, restart_unless_cancelled, concurrency_key,\n visible_to_runner_only, auto_kind, codebase, has_preprocessor,\n -- Same three forms and the same prefix-first rule as permissioned_as_exists,\n -- superadmin fallback included: one acting outside their workspaces has no usr\n -- row but still authenticates.\n CASE WHEN on_behalf_of LIKE 'u/%' THEN\n (SELECT on_behalf_of WHERE EXISTS (\n SELECT 1 FROM usr u WHERE u.workspace_id = $1::varchar\n AND u.username = substring(on_behalf_of from 3)\n UNION ALL\n SELECT 1 FROM password p WHERE p.super_admin\n AND (p.username = substring(on_behalf_of from 3)\n OR p.email = substring(on_behalf_of from 3))))\n WHEN on_behalf_of LIKE 'g/%' THEN\n (SELECT on_behalf_of WHERE EXISTS (\n SELECT 1 FROM group_ g WHERE g.workspace_id = $1::varchar\n AND g.name = substring(on_behalf_of from 3)))\n ELSE\n (SELECT on_behalf_of WHERE EXISTS (\n SELECT 1 FROM usr u WHERE u.workspace_id = $1::varchar\n AND u.username = on_behalf_of\n UNION ALL\n SELECT 1 FROM password p WHERE p.email = on_behalf_of\n AND p.super_admin))\n END, on_behalf_of_email, assets, modules, job_token_scopes\n FROM script\n WHERE workspace_id = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Text" + ] + }, + "nullable": [] + }, + "hash": "706f2227e34971d22a577b33f6aaa7f5755da9997419c4a9d07b30d4da318bea" +} diff --git a/backend/.sqlx/query-76440a1c5b40a897078817cc960247270c9044b363b71496ba36e1357c63a2a4.json b/backend/.sqlx/query-76440a1c5b40a897078817cc960247270c9044b363b71496ba36e1357c63a2a4.json new file mode 100644 index 0000000000..8eecbfa7bc --- /dev/null +++ b/backend/.sqlx/query-76440a1c5b40a897078817cc960247270c9044b363b71496ba36e1357c63a2a4.json @@ -0,0 +1,155 @@ +{ + "db_name": "PostgreSQL", + "query": "WITH RECURSIVE lineage AS (\n SELECT id, parent_job, kind, runnable_path, runnable_id, trigger_kind, trigger,\n tag, permissioned_as, args, 0 AS depth\n FROM v2_job WHERE id = $1 AND workspace_id = $2\n UNION ALL\n SELECT p.id, p.parent_job, p.kind, p.runnable_path, p.runnable_id, p.trigger_kind,\n p.trigger, p.tag, p.permissioned_as, p.args, l.depth + 1\n FROM v2_job p JOIN lineage l ON p.id = l.parent_job\n WHERE p.workspace_id = $2 AND l.depth < 100\n )\n SELECT id AS \"id!\", parent_job, kind AS \"kind!: JobKind\", runnable_path, runnable_id,\n trigger_kind::text AS trigger_kind, trigger, tag AS \"tag!\",\n permissioned_as AS \"permissioned_as!\",\n -- A restart takes its flow version from the request, so a trusted flow path\n -- can carry another flow's code: the version must belong to that path.\n CASE kind\n WHEN 'flow' THEN EXISTS (SELECT 1 FROM flow_version fv\n WHERE fv.id = runnable_id AND fv.path = runnable_path AND fv.workspace_id = $2)\n WHEN 'script' THEN EXISTS (SELECT 1 FROM script s\n WHERE s.hash = runnable_id AND s.path = runnable_path AND s.workspace_id = $2\n AND NOT s.deleted)\n ELSE true\n END AS \"origin_verified!\",\n -- What `run/f` and `run/p` resolve the path to now; for scripts the predicate of\n -- `get_latest_deployed_script_hash`.\n COALESCE(CASE kind\n WHEN 'flow' THEN runnable_id = (SELECT f.versions[array_upper(f.versions, 1)]\n FROM flow f WHERE f.path = runnable_path AND f.workspace_id = $2)\n WHEN 'script' THEN runnable_id = (SELECT s.hash FROM script s\n WHERE s.path = runnable_path AND s.workspace_id = $2 AND NOT s.deleted\n AND s.lock IS NOT NULL AND s.lock_error_logs IS NULL\n ORDER BY s.created_at DESC LIMIT 1)\n END, false) AS \"current_version!\",\n EXISTS (SELECT 1 FROM v2_job_status st WHERE st.id = lineage.id\n AND jsonb_typeof(st.flow_status->'restarted_from') = 'object')\n OR EXISTS (SELECT 1 FROM v2_job_completed c WHERE c.id = lineage.id\n AND jsonb_typeof(c.flow_status->'restarted_from') = 'object')\n AS \"restarted!\",\n COALESCE(\n (SELECT st.flow_status->'restarted_from'->>'flow_job_id' FROM v2_job_status st\n WHERE st.id = lineage.id),\n (SELECT c.flow_status->'restarted_from'->>'flow_job_id' FROM v2_job_completed c\n WHERE c.id = lineage.id)\n ) AS restarted_from,\n -- Only deployed-app runs are stamped with their app; an app editor preview\n -- runs app code at an app path it does not have to own.\n COALESCE(trigger_kind = 'app' AND starts_with(runnable_path, trigger || '/'), false)\n AS \"app_stamped!\",\n -- A preview's modules come from its args, which its parent may have taken from\n -- the caller.\n COALESCE(jsonb_typeof(args->'_MODULES') = 'object', false) AS \"args_modules!\",\n CASE WHEN depth = 0 THEN (SELECT wp.worker_group FROM v2_job_queue q\n JOIN worker_ping wp ON wp.worker = q.worker WHERE q.id = lineage.id)\n END AS worker_group,\n CASE WHEN depth = 0 THEN CASE\n WHEN starts_with(permissioned_as, 'g/') THEN 'group'\n ELSE (SELECT CASE WHEN u.is_service_account THEN 'service_account' ELSE 'user' END\n FROM usr u WHERE starts_with(permissioned_as, 'u/')\n AND u.username = substr(permissioned_as, 3) AND u.workspace_id = $2)\n END END AS run_as_type,\n -- The canonical item `item_digest` hashes, null members included.\n CASE WHEN depth = 0 OR depth = max(depth) OVER () THEN CASE kind\n WHEN 'script' THEN (SELECT jsonb_build_object('content', s.content,\n 'lock', s.lock, 'codebase', s.codebase,\n 'modules', (SELECT jsonb_object_agg(m.key, m.value - 'language')\n FROM jsonb_each(s.modules) m))\n FROM script s WHERE s.hash = runnable_id AND s.workspace_id = $2\n AND NOT s.deleted)\n WHEN 'flow' THEN (SELECT fv.value FROM flow_version fv\n WHERE fv.id = runnable_id AND fv.workspace_id = $2)\n WHEN 'flowscript' THEN (SELECT jsonb_build_object('content', n.code,\n 'lock', n.lock)\n FROM flow_node n WHERE n.id = runnable_id AND n.workspace_id = $2)\n END END AS digest_source\n FROM lineage ORDER BY depth", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id!", + "type_info": "Uuid" + }, + { + "ordinal": 1, + "name": "parent_job", + "type_info": "Uuid" + }, + { + "ordinal": 2, + "name": "kind!: JobKind", + "type_info": { + "Custom": { + "name": "job_kind", + "kind": { + "Enum": [ + "script", + "preview", + "flow", + "dependencies", + "flowpreview", + "script_hub", + "identity", + "flowdependencies", + "http", + "graphql", + "postgresql", + "noop", + "appdependencies", + "deploymentcallback", + "singlestepflow", + "flowscript", + "flownode", + "appscript", + "aiagent", + "unassigned_script", + "unassigned_flow", + "unassigned_singlestepflow" + ] + } + } + } + }, + { + "ordinal": 3, + "name": "runnable_path", + "type_info": "Varchar" + }, + { + "ordinal": 4, + "name": "runnable_id", + "type_info": "Int8" + }, + { + "ordinal": 5, + "name": "trigger_kind", + "type_info": "Text" + }, + { + "ordinal": 6, + "name": "trigger", + "type_info": "Varchar" + }, + { + "ordinal": 7, + "name": "tag!", + "type_info": "Varchar" + }, + { + "ordinal": 8, + "name": "permissioned_as!", + "type_info": "Varchar" + }, + { + "ordinal": 9, + "name": "origin_verified!", + "type_info": "Bool" + }, + { + "ordinal": 10, + "name": "current_version!", + "type_info": "Bool" + }, + { + "ordinal": 11, + "name": "restarted!", + "type_info": "Bool" + }, + { + "ordinal": 12, + "name": "restarted_from", + "type_info": "Text" + }, + { + "ordinal": 13, + "name": "app_stamped!", + "type_info": "Bool" + }, + { + "ordinal": 14, + "name": "args_modules!", + "type_info": "Bool" + }, + { + "ordinal": 15, + "name": "worker_group", + "type_info": "Varchar" + }, + { + "ordinal": 16, + "name": "run_as_type", + "type_info": "Text" + }, + { + "ordinal": 17, + "name": "digest_source", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Text" + ] + }, + "nullable": [ + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ] + }, + "hash": "76440a1c5b40a897078817cc960247270c9044b363b71496ba36e1357c63a2a4" +} diff --git a/backend/.sqlx/query-77e9cb16c25defc2423621a64f020e1e07f7e50df82fb6c33f79e6faa5b77bab.json b/backend/.sqlx/query-77e9cb16c25defc2423621a64f020e1e07f7e50df82fb6c33f79e6faa5b77bab.json new file mode 100644 index 0000000000..57822becd2 --- /dev/null +++ b/backend/.sqlx/query-77e9cb16c25defc2423621a64f020e1e07f7e50df82fb6c33f79e6faa5b77bab.json @@ -0,0 +1,29 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT coalesce(flow_version_lite.value, flow_version.value) as \"value!: sqlx::types::Json>\", flow.job_token_scopes FROM flow\n LEFT JOIN flow_version\n ON flow_version.id = flow.versions[array_upper(flow.versions, 1)]\n LEFT JOIN flow_version_lite\n ON flow_version_lite.id = flow_version.id\n WHERE flow.path = $1 AND flow.workspace_id = $2 LIMIT 1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "value!: sqlx::types::Json>", + "type_info": "Jsonb" + }, + { + "ordinal": 1, + "name": "job_token_scopes", + "type_info": "TextArray" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + null, + true + ] + }, + "hash": "77e9cb16c25defc2423621a64f020e1e07f7e50df82fb6c33f79e6faa5b77bab" +} diff --git a/backend/.sqlx/query-1debd472c9ffd2fc78877484f93db51f9aabed54f9894eda8ad610053ad76ce6.json b/backend/.sqlx/query-83e9c830ee816d4e4e7193a67823cab7c9a4461d3bfbdc0fe7dbdd28ae2de924.json similarity index 56% rename from backend/.sqlx/query-1debd472c9ffd2fc78877484f93db51f9aabed54f9894eda8ad610053ad76ce6.json rename to backend/.sqlx/query-83e9c830ee816d4e4e7193a67823cab7c9a4461d3bfbdc0fe7dbdd28ae2de924.json index 96147d531e..b6a5d2a2d0 100644 --- a/backend/.sqlx/query-1debd472c9ffd2fc78877484f93db51f9aabed54f9894eda8ad610053ad76ce6.json +++ b/backend/.sqlx/query-83e9c830ee816d4e4e7193a67823cab7c9a4461d3bfbdc0fe7dbdd28ae2de924.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT restart_unless_cancelled, timeout FROM script WHERE hash = $1 AND workspace_id = $2", + "query": "SELECT restart_unless_cancelled, timeout, job_token_scopes FROM script WHERE hash = $1 AND workspace_id = $2", "describe": { "columns": [ { @@ -12,6 +12,11 @@ "ordinal": 1, "name": "timeout", "type_info": "Int4" + }, + { + "ordinal": 2, + "name": "job_token_scopes", + "type_info": "TextArray" } ], "parameters": { @@ -21,9 +26,10 @@ ] }, "nullable": [ + true, true, true ] }, - "hash": "1debd472c9ffd2fc78877484f93db51f9aabed54f9894eda8ad610053ad76ce6" + "hash": "83e9c830ee816d4e4e7193a67823cab7c9a4461d3bfbdc0fe7dbdd28ae2de924" } diff --git a/backend/.sqlx/query-1860c102a5309a8e1bb7288d0616f48e4330af7fbe61426e999c6073f630d88f.json b/backend/.sqlx/query-8f5091626054a43be07768f30cdc99bd68f1de8064ace4155f4da2cd45d1b403.json similarity index 56% rename from backend/.sqlx/query-1860c102a5309a8e1bb7288d0616f48e4330af7fbe61426e999c6073f630d88f.json rename to backend/.sqlx/query-8f5091626054a43be07768f30cdc99bd68f1de8064ace4155f4da2cd45d1b403.json index d67ddf711b..574156394d 100644 --- a/backend/.sqlx/query-1860c102a5309a8e1bb7288d0616f48e4330af7fbe61426e999c6073f630d88f.json +++ b/backend/.sqlx/query-8f5091626054a43be07768f30cdc99bd68f1de8064ace4155f4da2cd45d1b403.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT DISTINCT ON (path)\n path,\n value as \"value!: sqlx::types::Json>\",\n created_at,\n typ::text as \"typ!\"\n FROM draft\n WHERE workspace_id = $1\n AND typ IN ('app', 'raw_app')\n AND (email = $2 OR email IS NULL)\n AND NOT EXISTS (\n SELECT 1 FROM app a\n WHERE a.workspace_id = draft.workspace_id\n AND a.path = draft.path\n )\n ORDER BY path, (email IS NULL), created_at DESC", + "query": "SELECT DISTINCT ON (path)\n path,\n value as \"value!: sqlx::types::Json>\",\n created_at,\n email IS NULL as \"legacy!\"\n FROM draft\n WHERE workspace_id = $1\n AND typ = 'flow'\n AND (email = $2 OR email IS NULL)\n AND NOT EXISTS (\n SELECT 1 FROM flow f\n WHERE f.workspace_id = draft.workspace_id\n AND f.path = draft.path\n )\n ORDER BY path, (email IS NULL)", "describe": { "columns": [ { @@ -20,8 +20,8 @@ }, { "ordinal": 3, - "name": "typ!", - "type_info": "Text" + "name": "legacy!", + "type_info": "Bool" } ], "parameters": { @@ -37,5 +37,5 @@ null ] }, - "hash": "1860c102a5309a8e1bb7288d0616f48e4330af7fbe61426e999c6073f630d88f" + "hash": "8f5091626054a43be07768f30cdc99bd68f1de8064ace4155f4da2cd45d1b403" } diff --git a/backend/.sqlx/query-fac99e27c8396185dd21c33baf649dd75b5f8ed859e410d2322ec3f565382fa8.json b/backend/.sqlx/query-9a29bd07a66f31f7414b79cafd8073194c6990eedad4b5b540134dd13d72abb1.json similarity index 52% rename from backend/.sqlx/query-fac99e27c8396185dd21c33baf649dd75b5f8ed859e410d2322ec3f565382fa8.json rename to backend/.sqlx/query-9a29bd07a66f31f7414b79cafd8073194c6990eedad4b5b540134dd13d72abb1.json index c624143ea4..e8f288f4a4 100644 --- a/backend/.sqlx/query-fac99e27c8396185dd21c33baf649dd75b5f8ed859e410d2322ec3f565382fa8.json +++ b/backend/.sqlx/query-9a29bd07a66f31f7414b79cafd8073194c6990eedad4b5b540134dd13d72abb1.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT DISTINCT ON (path)\n path,\n value as \"value!: sqlx::types::Json>\",\n created_at\n FROM draft\n WHERE workspace_id = $1\n AND typ = 'script'\n AND (email = $2 OR email IS NULL)\n AND NOT EXISTS (\n SELECT 1 FROM script s\n WHERE s.workspace_id = draft.workspace_id\n AND s.path = draft.path\n )\n ORDER BY path, (email IS NULL)", + "query": "SELECT DISTINCT ON (path)\n path,\n value as \"value!: sqlx::types::Json>\",\n created_at,\n email IS NULL as \"legacy!\"\n FROM draft\n WHERE workspace_id = $1\n AND typ = 'script'\n AND (email = $2 OR email IS NULL)\n AND NOT EXISTS (\n SELECT 1 FROM script s\n WHERE s.workspace_id = draft.workspace_id\n AND s.path = draft.path\n )\n ORDER BY path, (email IS NULL)", "describe": { "columns": [ { @@ -17,6 +17,11 @@ "ordinal": 2, "name": "created_at", "type_info": "Timestamptz" + }, + { + "ordinal": 3, + "name": "legacy!", + "type_info": "Bool" } ], "parameters": { @@ -28,8 +33,9 @@ "nullable": [ false, false, - false + false, + null ] }, - "hash": "fac99e27c8396185dd21c33baf649dd75b5f8ed859e410d2322ec3f565382fa8" + "hash": "9a29bd07a66f31f7414b79cafd8073194c6990eedad4b5b540134dd13d72abb1" } diff --git a/backend/.sqlx/query-12329c3359a7944ab5fa3aa27ddca1b26f340ccf574b9fa07641fe88b2d2987c.json b/backend/.sqlx/query-a6dafb34be6088886b078ed2b554de8c72e312ef8abd5ed29482a59c8ec51dd7.json similarity index 68% rename from backend/.sqlx/query-12329c3359a7944ab5fa3aa27ddca1b26f340ccf574b9fa07641fe88b2d2987c.json rename to backend/.sqlx/query-a6dafb34be6088886b078ed2b554de8c72e312ef8abd5ed29482a59c8ec51dd7.json index 4c120699aa..bde8b28723 100644 --- a/backend/.sqlx/query-12329c3359a7944ab5fa3aa27ddca1b26f340ccf574b9fa07641fe88b2d2987c.json +++ b/backend/.sqlx/query-a6dafb34be6088886b078ed2b554de8c72e312ef8abd5ed29482a59c8ec51dd7.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "INSERT INTO flow_conversation_message (conversation_id, message_type, content, job_id, step_name, success, tool_arguments, tool_result, reasoning, attachments)\n VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)", + "query": "INSERT INTO flow_conversation_message (conversation_id, message_type, content, job_id, turn_job_id, step_name, success, tool_arguments, tool_result, reasoning, attachments)\n VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11)", "describe": { "columns": [], "parameters": { @@ -20,6 +20,7 @@ }, "Text", "Uuid", + "Uuid", "Varchar", "Bool", "Text", @@ -30,5 +31,5 @@ }, "nullable": [] }, - "hash": "12329c3359a7944ab5fa3aa27ddca1b26f340ccf574b9fa07641fe88b2d2987c" + "hash": "a6dafb34be6088886b078ed2b554de8c72e312ef8abd5ed29482a59c8ec51dd7" } diff --git a/backend/.sqlx/query-df8586283178b0e684b9a0e359efb16e38fd68a5fe2b01cd77f74a725bbf37a3.json b/backend/.sqlx/query-aed606f8c8d91a924ac457b4eabdf391947d5fc3da90ef98a7b793c2430eb7bc.json similarity index 76% rename from backend/.sqlx/query-df8586283178b0e684b9a0e359efb16e38fd68a5fe2b01cd77f74a725bbf37a3.json rename to backend/.sqlx/query-aed606f8c8d91a924ac457b4eabdf391947d5fc3da90ef98a7b793c2430eb7bc.json index 3383951c13..0bd5d0eca5 100644 --- a/backend/.sqlx/query-df8586283178b0e684b9a0e359efb16e38fd68a5fe2b01cd77f74a725bbf37a3.json +++ b/backend/.sqlx/query-aed606f8c8d91a924ac457b4eabdf391947d5fc3da90ef98a7b793c2430eb7bc.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT q.id AS \"id!\", j.created_by, j.permissioned_as, j.permissioned_as_email, j.trigger, j.trigger_kind AS \"trigger_kind: TriggerKindLabel\", j.preprocessed, j.args AS \"args: sqlx::types::Json>>\" FROM v2_job_queue q JOIN v2_job j USING (id) JOIN script s ON s.workspace_id = j.workspace_id AND s.hash = j.runnable_id WHERE j.workspace_id = $1 AND j.runnable_path = $2 AND j.kind = 'script' AND j.flow_step_id IS NULL AND j.runnable_id != $3 AND q.canceled_by IS NULL AND s.restart_unless_cancelled", + "query": "SELECT q.id AS \"id!\", j.created_by, j.permissioned_as, j.permissioned_as_email, j.trigger, j.trigger_kind AS \"trigger_kind: TriggerKindLabel\", j.preprocessed, j.args AS \"args: sqlx::types::Json>>\", p.job_token_scopes AS \"job_token_scopes?\" FROM v2_job_queue q JOIN v2_job j USING (id) JOIN script s ON s.workspace_id = j.workspace_id AND s.hash = j.runnable_id LEFT JOIN job_perms p ON p.job_id = q.id WHERE j.workspace_id = $1 AND j.runnable_path = $2 AND j.kind = 'script' AND j.flow_step_id IS NULL AND j.runnable_id != $3 AND q.canceled_by IS NULL AND s.restart_unless_cancelled", "describe": { "columns": [ { @@ -71,6 +71,11 @@ "ordinal": 7, "name": "args: sqlx::types::Json>>", "type_info": "Jsonb" + }, + { + "ordinal": 8, + "name": "job_token_scopes?", + "type_info": "TextArray" } ], "parameters": { @@ -88,8 +93,9 @@ true, true, true, + true, true ] }, - "hash": "df8586283178b0e684b9a0e359efb16e38fd68a5fe2b01cd77f74a725bbf37a3" + "hash": "aed606f8c8d91a924ac457b4eabdf391947d5fc3da90ef98a7b793c2430eb7bc" } diff --git a/backend/.sqlx/query-b40c9e2d637cf0f2f49fd4c720b64eb6af24e8d8aa99b846f21f2ed49c320bdc.json b/backend/.sqlx/query-b40c9e2d637cf0f2f49fd4c720b64eb6af24e8d8aa99b846f21f2ed49c320bdc.json new file mode 100644 index 0000000000..db729b97b1 --- /dev/null +++ b/backend/.sqlx/query-b40c9e2d637cf0f2f49fd4c720b64eb6af24e8d8aa99b846f21f2ed49c320bdc.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO flow\n (workspace_id, path, summary, description, archived, extra_perms, dependency_job, tag, ws_error_handler_muted, dedicated_worker, timeout, visible_to_runner_only, on_behalf_of, on_behalf_of_email, concurrency_key, versions, value, schema, edited_by, edited_at, job_token_scopes)\n SELECT workspace_id, REGEXP_REPLACE(path,'u/' || $2 || '/(.*)','u/' || $1 || '/\\1'), summary, description, archived, extra_perms, dependency_job, tag, ws_error_handler_muted, dedicated_worker, timeout, visible_to_runner_only, on_behalf_of, on_behalf_of_email, concurrency_key, versions, value, schema, edited_by, edited_at, job_token_scopes\n FROM flow\n WHERE path LIKE ('u/' || $2 || '/%') AND workspace_id = $3", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "b40c9e2d637cf0f2f49fd4c720b64eb6af24e8d8aa99b846f21f2ed49c320bdc" +} diff --git a/backend/.sqlx/query-b48afe9913423a90955011fc6368b8517561f5b55a30e44801d6a9123a762584.json b/backend/.sqlx/query-b48afe9913423a90955011fc6368b8517561f5b55a30e44801d6a9123a762584.json new file mode 100644 index 0000000000..47dcca381c --- /dev/null +++ b/backend/.sqlx/query-b48afe9913423a90955011fc6368b8517561f5b55a30e44801d6a9123a762584.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT job_token_scopes FROM script WHERE path = $1 AND workspace_id = $2 AND deleted = false ORDER BY created_at DESC LIMIT 1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "job_token_scopes", + "type_info": "TextArray" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + true + ] + }, + "hash": "b48afe9913423a90955011fc6368b8517561f5b55a30e44801d6a9123a762584" +} diff --git a/backend/.sqlx/query-bfd01122256e922b00acf0e86593007c6aeccf76bfd07384ffaccf50a24d13d4.json b/backend/.sqlx/query-bfd01122256e922b00acf0e86593007c6aeccf76bfd07384ffaccf50a24d13d4.json new file mode 100644 index 0000000000..b1f969f343 --- /dev/null +++ b/backend/.sqlx/query-bfd01122256e922b00acf0e86593007c6aeccf76bfd07384ffaccf50a24d13d4.json @@ -0,0 +1,102 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO script (workspace_id, hash, path, parent_hashes, summary, description, content, created_by, schema, is_template, extra_perms, lock, language, kind, tag, envs, concurrent_limit, concurrency_time_window_s, cache_ttl, dedicated_worker, ws_error_handler_muted, priority, restart_unless_cancelled, delete_after_use, delete_after_secs, timeout, concurrency_key, visible_to_runner_only, auto_kind, codebase, has_preprocessor, schema_validation, assets, debounce_key, debounce_delay_s, cache_ignore_s3_path, runnable_settings_handle, modules, labels, on_behalf_of, on_behalf_of_email, job_token_scopes) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9::text::json, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26, $27, $28, $29, $30, $31, $32, $33, $34, $35, $36, $37, $38, $39, $40, $41, $42)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Int8", + "Varchar", + "Int8Array", + "Text", + "Text", + "Text", + "Varchar", + "Text", + "Bool", + "Jsonb", + "Text", + { + "Custom": { + "name": "script_lang", + "kind": { + "Enum": [ + "python3", + "deno", + "go", + "bash", + "postgresql", + "nativets", + "bun", + "mysql", + "bigquery", + "snowflake", + "graphql", + "powershell", + "mssql", + "php", + "bunnative", + "rust", + "ansible", + "csharp", + "oracledb", + "nu", + "java", + "duckdb", + "ruby", + "rlang", + "dbt" + ] + } + } + }, + { + "Custom": { + "name": "script_kind", + "kind": { + "Enum": [ + "script", + "trigger", + "failure", + "command", + "approval", + "preprocessor" + ] + } + } + }, + "Varchar", + "VarcharArray", + "Int4", + "Int4", + "Int4", + "Bool", + "Bool", + "Int2", + "Bool", + "Bool", + "Int4", + "Int4", + "Varchar", + "Bool", + "Varchar", + "Varchar", + "Bool", + "Bool", + "Jsonb", + "Varchar", + "Int4", + "Bool", + "Int8", + "Jsonb", + "TextArray", + "Varchar", + "Text", + "TextArray" + ] + }, + "nullable": [] + }, + "hash": "bfd01122256e922b00acf0e86593007c6aeccf76bfd07384ffaccf50a24d13d4" +} diff --git a/backend/.sqlx/query-c4eaa6e3b1d30a72ef1b54b2b7f28d84f1ca3c91de86d520dce06ac12d557bd5.json b/backend/.sqlx/query-c4eaa6e3b1d30a72ef1b54b2b7f28d84f1ca3c91de86d520dce06ac12d557bd5.json new file mode 100644 index 0000000000..151296d652 --- /dev/null +++ b/backend/.sqlx/query-c4eaa6e3b1d30a72ef1b54b2b7f28d84f1ca3c91de86d520dce06ac12d557bd5.json @@ -0,0 +1,24 @@ +{ + "db_name": "PostgreSQL", + "query": "WITH inserted AS (\n INSERT INTO flow\n (workspace_id, path, summary, description, archived, extra_perms, dependency_job, tag, ws_error_handler_muted, dedicated_worker, timeout, visible_to_runner_only, on_behalf_of, on_behalf_of_email, concurrency_key, versions, value, schema, edited_by, edited_at, labels, lock_error_logs, job_token_scopes)\n SELECT workspace_id, REGEXP_REPLACE(path, 'u/' || $2 || '/(.*)', $1 || '/\\1'), summary, description, archived, extra_perms, dependency_job, tag, ws_error_handler_muted, dedicated_worker, timeout, visible_to_runner_only, on_behalf_of, on_behalf_of_email, concurrency_key, versions, value, schema, edited_by, edited_at, labels, lock_error_logs, job_token_scopes\n FROM flow\n WHERE path LIKE ('u/' || $2 || '/%') AND workspace_id = $3\n RETURNING 1\n ) SELECT COUNT(*) FROM inserted", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "count", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "c4eaa6e3b1d30a72ef1b54b2b7f28d84f1ca3c91de86d520dce06ac12d557bd5" +} diff --git a/backend/.sqlx/query-c921b1a03e3181bbac351545941affc87853ff2d97f567a8817260cfef43e00e.json b/backend/.sqlx/query-c921b1a03e3181bbac351545941affc87853ff2d97f567a8817260cfef43e00e.json new file mode 100644 index 0000000000..393c88720d --- /dev/null +++ b/backend/.sqlx/query-c921b1a03e3181bbac351545941affc87853ff2d97f567a8817260cfef43e00e.json @@ -0,0 +1,65 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT email, username, is_admin, is_operator, groups, folders, end_user_email,\n job_token_scopes\n FROM job_perms WHERE job_id = $1 AND workspace_id = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "email", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "username", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "is_admin", + "type_info": "Bool" + }, + { + "ordinal": 3, + "name": "is_operator", + "type_info": "Bool" + }, + { + "ordinal": 4, + "name": "groups", + "type_info": "TextArray" + }, + { + "ordinal": 5, + "name": "folders", + "type_info": "JsonbArray" + }, + { + "ordinal": 6, + "name": "end_user_email", + "type_info": "Varchar" + }, + { + "ordinal": 7, + "name": "job_token_scopes", + "type_info": "TextArray" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Text" + ] + }, + "nullable": [ + false, + false, + false, + false, + false, + false, + true, + true + ] + }, + "hash": "c921b1a03e3181bbac351545941affc87853ff2d97f567a8817260cfef43e00e" +} diff --git a/backend/.sqlx/query-ce3bccee14cd107631c4ddb4a467099be925f1058c76c6af31e8a620607cf6a1.json b/backend/.sqlx/query-ce3bccee14cd107631c4ddb4a467099be925f1058c76c6af31e8a620607cf6a1.json new file mode 100644 index 0000000000..4ae34431dc --- /dev/null +++ b/backend/.sqlx/query-ce3bccee14cd107631c4ddb4a467099be925f1058c76c6af31e8a620607cf6a1.json @@ -0,0 +1,30 @@ +{ + "db_name": "PostgreSQL", + "query": "\n UPDATE\n flow\n SET\n path = $1,\n summary = $2,\n description = $3,\n dependency_job = NULL,\n lock_error_logs = '',\n tag = $4,\n dedicated_worker = $5,\n visible_to_runner_only = $6,\n ws_error_handler_muted = $7,\n value = $8,\n schema = $9::text::json,\n edited_by = $10,\n edited_at = now(),\n labels = COALESCE($13, labels),\n on_behalf_of = $14,\n on_behalf_of_email = $15,\n job_token_scopes = CASE WHEN $16 THEN $17 ELSE job_token_scopes END\n WHERE\n path = $11 AND workspace_id = $12", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Text", + "Text", + "Varchar", + "Bool", + "Bool", + "Bool", + "Jsonb", + "Text", + "Varchar", + "Text", + "Text", + "TextArray", + "Varchar", + "Text", + "Bool", + "TextArray" + ] + }, + "nullable": [] + }, + "hash": "ce3bccee14cd107631c4ddb4a467099be925f1058c76c6af31e8a620607cf6a1" +} diff --git a/backend/.sqlx/query-d51f555e80b27549b884f334087878286dcc2f23101c2e1c8c1d07b6f0deaecd.json b/backend/.sqlx/query-d51f555e80b27549b884f334087878286dcc2f23101c2e1c8c1d07b6f0deaecd.json new file mode 100644 index 0000000000..ae90b163ff --- /dev/null +++ b/backend/.sqlx/query-d51f555e80b27549b884f334087878286dcc2f23101c2e1c8c1d07b6f0deaecd.json @@ -0,0 +1,159 @@ +{ + "db_name": "PostgreSQL", + "query": "select hash, tag, concurrency_key, concurrent_limit, concurrency_time_window_s, debounce_key, debounce_delay_s, cache_ttl, cache_ignore_s3_path, runnable_settings_handle, language as \"language: ScriptLang\", dedicated_worker, priority, timeout, on_behalf_of, created_by, labels, job_token_scopes FROM script\n WHERE path = $1 AND workspace_id = $2 AND archived = false AND (lock IS NOT NULL OR $3 = false)\n ORDER BY created_at DESC LIMIT 1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "hash", + "type_info": "Int8" + }, + { + "ordinal": 1, + "name": "tag", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "concurrency_key", + "type_info": "Varchar" + }, + { + "ordinal": 3, + "name": "concurrent_limit", + "type_info": "Int4" + }, + { + "ordinal": 4, + "name": "concurrency_time_window_s", + "type_info": "Int4" + }, + { + "ordinal": 5, + "name": "debounce_key", + "type_info": "Varchar" + }, + { + "ordinal": 6, + "name": "debounce_delay_s", + "type_info": "Int4" + }, + { + "ordinal": 7, + "name": "cache_ttl", + "type_info": "Int4" + }, + { + "ordinal": 8, + "name": "cache_ignore_s3_path", + "type_info": "Bool" + }, + { + "ordinal": 9, + "name": "runnable_settings_handle", + "type_info": "Int8" + }, + { + "ordinal": 10, + "name": "language: ScriptLang", + "type_info": { + "Custom": { + "name": "script_lang", + "kind": { + "Enum": [ + "python3", + "deno", + "go", + "bash", + "postgresql", + "nativets", + "bun", + "mysql", + "bigquery", + "snowflake", + "graphql", + "powershell", + "mssql", + "php", + "bunnative", + "rust", + "ansible", + "csharp", + "oracledb", + "nu", + "java", + "duckdb", + "ruby", + "rlang", + "dbt" + ] + } + } + } + }, + { + "ordinal": 11, + "name": "dedicated_worker", + "type_info": "Bool" + }, + { + "ordinal": 12, + "name": "priority", + "type_info": "Int2" + }, + { + "ordinal": 13, + "name": "timeout", + "type_info": "Int4" + }, + { + "ordinal": 14, + "name": "on_behalf_of", + "type_info": "Varchar" + }, + { + "ordinal": 15, + "name": "created_by", + "type_info": "Varchar" + }, + { + "ordinal": 16, + "name": "labels", + "type_info": "TextArray" + }, + { + "ordinal": 17, + "name": "job_token_scopes", + "type_info": "TextArray" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Bool" + ] + }, + "nullable": [ + false, + true, + true, + true, + true, + true, + true, + true, + true, + true, + false, + true, + true, + true, + true, + false, + true, + true + ] + }, + "hash": "d51f555e80b27549b884f334087878286dcc2f23101c2e1c8c1d07b6f0deaecd" +} diff --git a/backend/.sqlx/query-967f52005f4a044b3a2e9f02ceadf90dad5246681bde6caaa633b85a5e8b2352.json b/backend/.sqlx/query-d5f1e272af4a428daedafa3453124b1973a48d47c1c563b2213055bfcf688230.json similarity index 57% rename from backend/.sqlx/query-967f52005f4a044b3a2e9f02ceadf90dad5246681bde6caaa633b85a5e8b2352.json rename to backend/.sqlx/query-d5f1e272af4a428daedafa3453124b1973a48d47c1c563b2213055bfcf688230.json index 5161f716ff..190c14788f 100644 --- a/backend/.sqlx/query-967f52005f4a044b3a2e9f02ceadf90dad5246681bde6caaa633b85a5e8b2352.json +++ b/backend/.sqlx/query-d5f1e272af4a428daedafa3453124b1973a48d47c1c563b2213055bfcf688230.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "DELETE FROM flow_conversation_message WHERE job_id = ANY($1) RETURNING conversation_id", + "query": "DELETE FROM flow_conversation_message WHERE COALESCE(turn_job_id, job_id) = ANY($1)\n RETURNING conversation_id", "describe": { "columns": [ { @@ -18,5 +18,5 @@ false ] }, - "hash": "967f52005f4a044b3a2e9f02ceadf90dad5246681bde6caaa633b85a5e8b2352" + "hash": "d5f1e272af4a428daedafa3453124b1973a48d47c1c563b2213055bfcf688230" } diff --git a/backend/.sqlx/query-ebce1633afe3d3f8a1a68bc8d26adcdd88b137bb892437a3ca67fd716772512b.json b/backend/.sqlx/query-d61e8523269a26051079b0ee837bf33c1ce2e063aaf80fd155dbbbf99eb2fab3.json similarity index 59% rename from backend/.sqlx/query-ebce1633afe3d3f8a1a68bc8d26adcdd88b137bb892437a3ca67fd716772512b.json rename to backend/.sqlx/query-d61e8523269a26051079b0ee837bf33c1ce2e063aaf80fd155dbbbf99eb2fab3.json index 1930427b94..7b1e8d29cc 100644 --- a/backend/.sqlx/query-ebce1633afe3d3f8a1a68bc8d26adcdd88b137bb892437a3ca67fd716772512b.json +++ b/backend/.sqlx/query-d61e8523269a26051079b0ee837bf33c1ce2e063aaf80fd155dbbbf99eb2fab3.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT\n COUNT(*) FILTER (WHERE NOT operator AND NOT disabled AND NOT is_service_account) AS \"developers!\",\n COUNT(*) FILTER (WHERE operator AND NOT disabled AND NOT is_service_account) AS \"operators!\"\n FROM usr WHERE workspace_id = $1", + "query": "SELECT\n COUNT(*) FILTER (WHERE NOT operator AND NOT disabled AND NOT is_service_account) AS \"developers!\",\n COUNT(*) FILTER (WHERE operator AND NOT disabled AND NOT is_service_account) AS \"operators!\",\n COUNT(*) FILTER (WHERE is_service_account AND NOT disabled) AS \"service_accounts!\"\n FROM usr WHERE workspace_id = $1", "describe": { "columns": [ { @@ -12,6 +12,11 @@ "ordinal": 1, "name": "operators!", "type_info": "Int8" + }, + { + "ordinal": 2, + "name": "service_accounts!", + "type_info": "Int8" } ], "parameters": { @@ -20,9 +25,10 @@ ] }, "nullable": [ + null, null, null ] }, - "hash": "ebce1633afe3d3f8a1a68bc8d26adcdd88b137bb892437a3ca67fd716772512b" + "hash": "d61e8523269a26051079b0ee837bf33c1ce2e063aaf80fd155dbbbf99eb2fab3" } diff --git a/backend/.sqlx/query-df1fe2e7ef004b8de0e626f40fd737b3b4b662a89148cf6ac70eb3563ad7f0c5.json b/backend/.sqlx/query-df1fe2e7ef004b8de0e626f40fd737b3b4b662a89148cf6ac70eb3563ad7f0c5.json new file mode 100644 index 0000000000..7f160dc856 --- /dev/null +++ b/backend/.sqlx/query-df1fe2e7ef004b8de0e626f40fd737b3b4b662a89148cf6ac70eb3563ad7f0c5.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO flow (\n workspace_id, path, summary, description, value, edited_by, edited_at,\n archived, schema, extra_perms, dependency_job, tag,\n ws_error_handler_muted, dedicated_worker, timeout, visible_to_runner_only,\n concurrency_key, versions, on_behalf_of, on_behalf_of_email, lock_error_logs,\n job_token_scopes\n )\n SELECT $2, path, summary, description, value, edited_by, edited_at,\n archived, schema, extra_perms, NULL, tag,\n ws_error_handler_muted, dedicated_worker, timeout, visible_to_runner_only,\n concurrency_key, ARRAY[]::bigint[],\n -- Same predicate as clone_scripts.\n CASE WHEN on_behalf_of LIKE 'u/%' THEN\n (SELECT on_behalf_of WHERE EXISTS (\n SELECT 1 FROM usr u WHERE u.workspace_id = $2::varchar\n AND u.username = substring(on_behalf_of from 3)\n UNION ALL\n SELECT 1 FROM password p WHERE p.super_admin\n AND (p.username = substring(on_behalf_of from 3)\n OR p.email = substring(on_behalf_of from 3))))\n WHEN on_behalf_of LIKE 'g/%' THEN\n (SELECT on_behalf_of WHERE EXISTS (\n SELECT 1 FROM group_ g WHERE g.workspace_id = $2::varchar\n AND g.name = substring(on_behalf_of from 3)))\n ELSE\n (SELECT on_behalf_of WHERE EXISTS (\n SELECT 1 FROM usr u WHERE u.workspace_id = $2::varchar\n AND u.username = on_behalf_of\n UNION ALL\n SELECT 1 FROM password p WHERE p.email = on_behalf_of\n AND p.super_admin))\n END, on_behalf_of_email, lock_error_logs, job_token_scopes\n FROM flow\n WHERE workspace_id = $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "df1fe2e7ef004b8de0e626f40fd737b3b4b662a89148cf6ac70eb3563ad7f0c5" +} diff --git a/backend/.sqlx/query-e81eaa501b4af1269a4929ec660d10fc10d9ec4dc2359f43d79fe255d101cad8.json b/backend/.sqlx/query-e81eaa501b4af1269a4929ec660d10fc10d9ec4dc2359f43d79fe255d101cad8.json new file mode 100644 index 0000000000..909565b8c6 --- /dev/null +++ b/backend/.sqlx/query-e81eaa501b4af1269a4929ec660d10fc10d9ec4dc2359f43d79fe255d101cad8.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO flow\n (workspace_id, path, summary, description, archived, extra_perms, dependency_job, tag, ws_error_handler_muted, dedicated_worker, timeout, visible_to_runner_only, on_behalf_of, on_behalf_of_email, concurrency_key, versions, value, schema, edited_by, edited_at, lock_error_logs, job_token_scopes)\n SELECT $1, path, summary, description, archived, extra_perms, dependency_job, tag, ws_error_handler_muted, dedicated_worker, timeout, visible_to_runner_only, on_behalf_of, on_behalf_of_email, concurrency_key, versions, value, schema, edited_by, edited_at, lock_error_logs, job_token_scopes\n FROM flow WHERE workspace_id = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Text" + ] + }, + "nullable": [] + }, + "hash": "e81eaa501b4af1269a4929ec660d10fc10d9ec4dc2359f43d79fe255d101cad8" +} diff --git a/backend/.sqlx/query-ee92100fcd681515ba81f23970e64af7f33464879bef4d1eff682126dffeef64.json b/backend/.sqlx/query-ee92100fcd681515ba81f23970e64af7f33464879bef4d1eff682126dffeef64.json deleted file mode 100644 index 3c52f8ba96..0000000000 --- a/backend/.sqlx/query-ee92100fcd681515ba81f23970e64af7f33464879bef4d1eff682126dffeef64.json +++ /dev/null @@ -1,125 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "WITH RECURSIVE lineage AS (\n SELECT id, parent_job, kind, runnable_path, runnable_id, trigger_kind, trigger,\n permissioned_as, args, 0 AS depth\n FROM v2_job WHERE id = $1 AND workspace_id = $2\n UNION ALL\n SELECT p.id, p.parent_job, p.kind, p.runnable_path, p.runnable_id, p.trigger_kind,\n p.trigger, p.permissioned_as, p.args, l.depth + 1\n FROM v2_job p JOIN lineage l ON p.id = l.parent_job\n WHERE p.workspace_id = $2 AND l.depth < 100\n )\n SELECT id AS \"id!\", parent_job, kind AS \"kind!: JobKind\", runnable_path, runnable_id,\n trigger_kind::text AS trigger_kind, permissioned_as AS \"permissioned_as!\",\n -- A restart takes its flow version from the request, so a trusted flow path\n -- can carry another flow's code: the version must belong to that path.\n CASE kind\n WHEN 'flow' THEN EXISTS (SELECT 1 FROM flow_version fv\n WHERE fv.id = runnable_id AND fv.path = runnable_path AND fv.workspace_id = $2)\n WHEN 'script' THEN EXISTS (SELECT 1 FROM script s\n WHERE s.hash = runnable_id AND s.path = runnable_path AND s.workspace_id = $2\n AND NOT s.deleted)\n ELSE true\n END AS \"origin_verified!\",\n -- What `run/f` and `run/p` resolve the path to now; for scripts the predicate of\n -- `get_latest_deployed_script_hash`.\n COALESCE(CASE kind\n WHEN 'flow' THEN runnable_id = (SELECT f.versions[array_upper(f.versions, 1)]\n FROM flow f WHERE f.path = runnable_path AND f.workspace_id = $2)\n WHEN 'script' THEN runnable_id = (SELECT s.hash FROM script s\n WHERE s.path = runnable_path AND s.workspace_id = $2 AND NOT s.deleted\n AND s.lock IS NOT NULL AND s.lock_error_logs IS NULL\n ORDER BY s.created_at DESC LIMIT 1)\n END, false) AS \"current_version!\",\n EXISTS (SELECT 1 FROM v2_job_status st WHERE st.id = lineage.id\n AND jsonb_typeof(st.flow_status->'restarted_from') = 'object')\n OR EXISTS (SELECT 1 FROM v2_job_completed c WHERE c.id = lineage.id\n AND jsonb_typeof(c.flow_status->'restarted_from') = 'object')\n AS \"restarted!\",\n COALESCE(\n (SELECT st.flow_status->'restarted_from'->>'flow_job_id' FROM v2_job_status st\n WHERE st.id = lineage.id),\n (SELECT c.flow_status->'restarted_from'->>'flow_job_id' FROM v2_job_completed c\n WHERE c.id = lineage.id)\n ) AS restarted_from,\n -- Only deployed-app runs are stamped with their app; an app editor preview\n -- runs app code at an app path it does not have to own.\n COALESCE(trigger_kind = 'app' AND starts_with(runnable_path, trigger || '/'), false)\n AS \"app_stamped!\",\n -- A preview's modules come from its args, which its parent may have taken from\n -- the caller.\n COALESCE(jsonb_typeof(args->'_MODULES') = 'object', false) AS \"args_modules!\"\n FROM lineage ORDER BY depth", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "id!", - "type_info": "Uuid" - }, - { - "ordinal": 1, - "name": "parent_job", - "type_info": "Uuid" - }, - { - "ordinal": 2, - "name": "kind!: JobKind", - "type_info": { - "Custom": { - "name": "job_kind", - "kind": { - "Enum": [ - "script", - "preview", - "flow", - "dependencies", - "flowpreview", - "script_hub", - "identity", - "flowdependencies", - "http", - "graphql", - "postgresql", - "noop", - "appdependencies", - "deploymentcallback", - "singlestepflow", - "flowscript", - "flownode", - "appscript", - "aiagent", - "unassigned_script", - "unassigned_flow", - "unassigned_singlestepflow" - ] - } - } - } - }, - { - "ordinal": 3, - "name": "runnable_path", - "type_info": "Varchar" - }, - { - "ordinal": 4, - "name": "runnable_id", - "type_info": "Int8" - }, - { - "ordinal": 5, - "name": "trigger_kind", - "type_info": "Text" - }, - { - "ordinal": 6, - "name": "permissioned_as!", - "type_info": "Varchar" - }, - { - "ordinal": 7, - "name": "origin_verified!", - "type_info": "Bool" - }, - { - "ordinal": 8, - "name": "current_version!", - "type_info": "Bool" - }, - { - "ordinal": 9, - "name": "restarted!", - "type_info": "Bool" - }, - { - "ordinal": 10, - "name": "restarted_from", - "type_info": "Text" - }, - { - "ordinal": 11, - "name": "app_stamped!", - "type_info": "Bool" - }, - { - "ordinal": 12, - "name": "args_modules!", - "type_info": "Bool" - } - ], - "parameters": { - "Left": [ - "Uuid", - "Text" - ] - }, - "nullable": [ - null, - null, - null, - null, - null, - null, - null, - null, - null, - null, - null, - null, - null - ] - }, - "hash": "ee92100fcd681515ba81f23970e64af7f33464879bef4d1eff682126dffeef64" -} diff --git a/backend/.sqlx/query-eefb19c05e6d9650b4dd1e50e10c26f01e117b75ab48e197c7d91d388865da74.json b/backend/.sqlx/query-eefb19c05e6d9650b4dd1e50e10c26f01e117b75ab48e197c7d91d388865da74.json new file mode 100644 index 0000000000..24c87414b1 --- /dev/null +++ b/backend/.sqlx/query-eefb19c05e6d9650b4dd1e50e10c26f01e117b75ab48e197c7d91d388865da74.json @@ -0,0 +1,398 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT\n v2_job.id,\n v2_job.raw_code,\n v2_job.raw_lock,\n v2_job.raw_flow as \"raw_flow: _\",\n v2_job.tag,\n v2_job.created_at,\n v2_job.created_by,\n v2_job.permissioned_as,\n v2_job.permissioned_as_email,\n v2_job.kind as \"kind: _\",\n v2_job.runnable_id,\n v2_job.runnable_path,\n v2_job.parent_job,\n v2_job.root_job,\n v2_job.script_lang as \"script_lang: _\",\n v2_job.script_entrypoint_override,\n v2_job.flow_step,\n v2_job.flow_step_id,\n v2_job.flow_innermost_root_job,\n v2_job.\"trigger\",\n v2_job.trigger_kind as \"trigger_kind: _\",\n v2_job.same_worker,\n v2_job.visible_to_owner,\n v2_job.concurrent_limit,\n v2_job.concurrency_time_window_s,\n v2_job.cache_ttl,\n v2_job.timeout,\n v2_job.priority,\n v2_job.preprocessed,\n v2_job.args as \"args: _\",\n v2_job.labels,\n job_perms.job_token_scopes as \"job_token_scopes?\",\n v2_job.pre_run_error,\n\n v2_job_queue.started_at,\n v2_job_queue.scheduled_for,\n v2_job_queue.running,\n v2_job_queue.canceled_by,\n v2_job_queue.canceled_reason,\n v2_job_queue.suspend,\n v2_job_queue.suspend_until,\n v2_job_queue.worker,\n v2_job_queue.extras as \"extras: _\",\n\n v2_job_runtime.ping,\n v2_job_runtime.memory_peak,\n\n v2_job_status.flow_status as \"flow_status: _\",\n v2_job_status.flow_leaf_jobs as \"flow_leaf_jobs: _\",\n v2_job_status.workflow_as_code_status as \"workflow_as_code_status: _\",\n\n concurrency_key.key as \"concurrency_key?\"\n FROM v2_job_queue\n INNER JOIN v2_job ON v2_job.id = v2_job_queue.id\n LEFT JOIN job_perms ON job_perms.job_id = v2_job_queue.id\n LEFT JOIN v2_job_runtime ON v2_job_runtime.id = v2_job_queue.id\n LEFT JOIN v2_job_status ON v2_job_status.id = v2_job_queue.id\n LEFT JOIN concurrency_key ON concurrency_key.job_id = v2_job_queue.id\n WHERE v2_job_queue.workspace_id = $1\n AND v2_job_queue.running = false\n AND v2_job.parent_job IS NULL\n AND v2_job.trigger_kind IS DISTINCT FROM 'schedule'\n ORDER BY v2_job.created_at DESC\n LIMIT $2\n OFFSET $3\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Uuid" + }, + { + "ordinal": 1, + "name": "raw_code", + "type_info": "Text" + }, + { + "ordinal": 2, + "name": "raw_lock", + "type_info": "Text" + }, + { + "ordinal": 3, + "name": "raw_flow: _", + "type_info": "Jsonb" + }, + { + "ordinal": 4, + "name": "tag", + "type_info": "Varchar" + }, + { + "ordinal": 5, + "name": "created_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 6, + "name": "created_by", + "type_info": "Varchar" + }, + { + "ordinal": 7, + "name": "permissioned_as", + "type_info": "Varchar" + }, + { + "ordinal": 8, + "name": "permissioned_as_email", + "type_info": "Varchar" + }, + { + "ordinal": 9, + "name": "kind: _", + "type_info": { + "Custom": { + "name": "job_kind", + "kind": { + "Enum": [ + "script", + "preview", + "flow", + "dependencies", + "flowpreview", + "script_hub", + "identity", + "flowdependencies", + "http", + "graphql", + "postgresql", + "noop", + "appdependencies", + "deploymentcallback", + "singlestepflow", + "flowscript", + "flownode", + "appscript", + "aiagent", + "unassigned_script", + "unassigned_flow", + "unassigned_singlestepflow" + ] + } + } + } + }, + { + "ordinal": 10, + "name": "runnable_id", + "type_info": "Int8" + }, + { + "ordinal": 11, + "name": "runnable_path", + "type_info": "Varchar" + }, + { + "ordinal": 12, + "name": "parent_job", + "type_info": "Uuid" + }, + { + "ordinal": 13, + "name": "root_job", + "type_info": "Uuid" + }, + { + "ordinal": 14, + "name": "script_lang: _", + "type_info": { + "Custom": { + "name": "script_lang", + "kind": { + "Enum": [ + "python3", + "deno", + "go", + "bash", + "postgresql", + "nativets", + "bun", + "mysql", + "bigquery", + "snowflake", + "graphql", + "powershell", + "mssql", + "php", + "bunnative", + "rust", + "ansible", + "csharp", + "oracledb", + "nu", + "java", + "duckdb", + "ruby", + "rlang", + "dbt" + ] + } + } + } + }, + { + "ordinal": 15, + "name": "script_entrypoint_override", + "type_info": "Varchar" + }, + { + "ordinal": 16, + "name": "flow_step", + "type_info": "Int4" + }, + { + "ordinal": 17, + "name": "flow_step_id", + "type_info": "Varchar" + }, + { + "ordinal": 18, + "name": "flow_innermost_root_job", + "type_info": "Uuid" + }, + { + "ordinal": 19, + "name": "trigger", + "type_info": "Varchar" + }, + { + "ordinal": 20, + "name": "trigger_kind: _", + "type_info": { + "Custom": { + "name": "job_trigger_kind", + "kind": { + "Enum": [ + "webhook", + "http", + "websocket", + "kafka", + "email", + "nats", + "schedule", + "app", + "ui", + "postgres", + "sqs", + "gcp", + "mqtt", + "nextcloud", + "google", + "ci_test", + "github", + "azure", + "asset", + "freshness", + "amqp" + ] + } + } + } + }, + { + "ordinal": 21, + "name": "same_worker", + "type_info": "Bool" + }, + { + "ordinal": 22, + "name": "visible_to_owner", + "type_info": "Bool" + }, + { + "ordinal": 23, + "name": "concurrent_limit", + "type_info": "Int4" + }, + { + "ordinal": 24, + "name": "concurrency_time_window_s", + "type_info": "Int4" + }, + { + "ordinal": 25, + "name": "cache_ttl", + "type_info": "Int4" + }, + { + "ordinal": 26, + "name": "timeout", + "type_info": "Int4" + }, + { + "ordinal": 27, + "name": "priority", + "type_info": "Int2" + }, + { + "ordinal": 28, + "name": "preprocessed", + "type_info": "Bool" + }, + { + "ordinal": 29, + "name": "args: _", + "type_info": "Jsonb" + }, + { + "ordinal": 30, + "name": "labels", + "type_info": "TextArray" + }, + { + "ordinal": 31, + "name": "job_token_scopes?", + "type_info": "TextArray" + }, + { + "ordinal": 32, + "name": "pre_run_error", + "type_info": "Text" + }, + { + "ordinal": 33, + "name": "started_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 34, + "name": "scheduled_for", + "type_info": "Timestamptz" + }, + { + "ordinal": 35, + "name": "running", + "type_info": "Bool" + }, + { + "ordinal": 36, + "name": "canceled_by", + "type_info": "Varchar" + }, + { + "ordinal": 37, + "name": "canceled_reason", + "type_info": "Text" + }, + { + "ordinal": 38, + "name": "suspend", + "type_info": "Int4" + }, + { + "ordinal": 39, + "name": "suspend_until", + "type_info": "Timestamptz" + }, + { + "ordinal": 40, + "name": "worker", + "type_info": "Varchar" + }, + { + "ordinal": 41, + "name": "extras: _", + "type_info": "Jsonb" + }, + { + "ordinal": 42, + "name": "ping", + "type_info": "Timestamptz" + }, + { + "ordinal": 43, + "name": "memory_peak", + "type_info": "Int4" + }, + { + "ordinal": 44, + "name": "flow_status: _", + "type_info": "Jsonb" + }, + { + "ordinal": 45, + "name": "flow_leaf_jobs: _", + "type_info": "Jsonb" + }, + { + "ordinal": 46, + "name": "workflow_as_code_status: _", + "type_info": "Jsonb" + }, + { + "ordinal": 47, + "name": "concurrency_key?", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text", + "Int8", + "Int8" + ] + }, + "nullable": [ + false, + true, + true, + true, + false, + false, + false, + false, + false, + false, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + false, + false, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + false, + false, + true, + true, + false, + true, + true, + true, + true, + true, + true, + true, + true, + false + ] + }, + "hash": "eefb19c05e6d9650b4dd1e50e10c26f01e117b75ab48e197c7d91d388865da74" +} diff --git a/backend/.sqlx/query-f2146082f64fcdb4acdfe92e42f084f90b3ac06139ffca98670215956150563a.json b/backend/.sqlx/query-f2146082f64fcdb4acdfe92e42f084f90b3ac06139ffca98670215956150563a.json new file mode 100644 index 0000000000..ad4991a7cf --- /dev/null +++ b/backend/.sqlx/query-f2146082f64fcdb4acdfe92e42f084f90b3ac06139ffca98670215956150563a.json @@ -0,0 +1,28 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO flow (\n workspace_id, path, summary, description,\n dependency_job, lock_error_logs, tag,\n dedicated_worker, visible_to_runner_only,\n ws_error_handler_muted,\n value, schema, edited_by, edited_at, labels,\n on_behalf_of, on_behalf_of_email, job_token_scopes\n ) VALUES (\n $1, $2, $3, $4,\n NULL, '', $5,\n $6, $7,\n $8,\n $9, $10::text::json, $11, now(), $12,\n $13, $14, $15\n )", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Text", + "Text", + "Varchar", + "Bool", + "Bool", + "Bool", + "Jsonb", + "Text", + "Varchar", + "TextArray", + "Varchar", + "Text", + "TextArray" + ] + }, + "nullable": [] + }, + "hash": "f2146082f64fcdb4acdfe92e42f084f90b3ac06139ffca98670215956150563a" +} diff --git a/backend/.sqlx/query-69bfbe9b39414b724488532cc3b3659915d9fcb3d58f16532aaffe06c44ec976.json b/backend/.sqlx/query-fa81e10f725586e12c77d4f6387cc76d5569621612ade0270261c172102015b4.json similarity index 67% rename from backend/.sqlx/query-69bfbe9b39414b724488532cc3b3659915d9fcb3d58f16532aaffe06c44ec976.json rename to backend/.sqlx/query-fa81e10f725586e12c77d4f6387cc76d5569621612ade0270261c172102015b4.json index 52105ca605..6fbc58c626 100644 --- a/backend/.sqlx/query-69bfbe9b39414b724488532cc3b3659915d9fcb3d58f16532aaffe06c44ec976.json +++ b/backend/.sqlx/query-fa81e10f725586e12c77d4f6387cc76d5569621612ade0270261c172102015b4.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "DELETE FROM flow_conversation_message m\n USING flow_conversation c\n WHERE m.conversation_id = c.id AND c.workspace_id = $1 AND m.job_id = ANY($2)\n RETURNING m.conversation_id", + "query": "DELETE FROM flow_conversation_message m\n USING flow_conversation c\n WHERE m.conversation_id = c.id AND c.workspace_id = $1\n AND COALESCE(m.turn_job_id, m.job_id) = ANY($2)\n RETURNING m.conversation_id", "describe": { "columns": [ { @@ -19,5 +19,5 @@ false ] }, - "hash": "69bfbe9b39414b724488532cc3b3659915d9fcb3d58f16532aaffe06c44ec976" + "hash": "fa81e10f725586e12c77d4f6387cc76d5569621612ade0270261c172102015b4" } diff --git a/backend/Cargo.lock b/backend/Cargo.lock index abc735f0c7..76047b4909 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -828,13 +828,13 @@ dependencies = [ [[package]] name = "async-recursion" -version = "1.1.1" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b43422f69d8ff38f95f1b2bb76517c91589a924d1559a0e935d7c8ce0274c11" +checksum = "4f8abc12baad266b1c8cec146854c195b5864b4221d4b2ca7296a7ae82d9e451" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn 3.0.6", ] [[package]] @@ -1254,7 +1254,7 @@ dependencies = [ "bytes", "form_urlencoded", "hex", - "hmac", + "hmac 0.12.1", "http 0.2.12", "http 1.5.0", "percent-encoding", @@ -1963,18 +1963,18 @@ dependencies = [ [[package]] name = "bon" -version = "3.10.1" +version = "3.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60eafe0d77c3a2fc292c1d1346c3041b33c0a108085a2afabf672b70f69dbbc9" +checksum = "214f5df094ce551a10a30ffb9c70243d61f121a3d985a6495933e181dee6a7d2" dependencies = [ "bon-macros", ] [[package]] name = "bon-macros" -version = "3.10.1" +version = "3.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd0f9631d8aaaee112c41985d675ef269e02acbd4f33122836af4f0c5f699ff6" +checksum = "2706da6c749998cc555d04184909608956a69ad3f8ab1a076fbc867b4a3c3ce5" dependencies = [ "darling 0.24.1", "ident_case", @@ -2316,9 +2316,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.5.1" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f360145194ee8e21db5ee7f3fcd4fe52210864c75c985dae33218202c8bbe040" +checksum = "f74872d07caf508b30a21f6836e7d7016a2eaf7d9ff4f48deaa58cd8a0407630" dependencies = [ "find-msvc-tools", "jobserver", @@ -2489,7 +2489,7 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7b77c319abfd5219629c45c34c89ba945ed3c5e49fcde9d16b6c3885f118a730" dependencies = [ - "const-oid", + "const-oid 0.9.6", "der", "spki", "x509-cert", @@ -2568,6 +2568,12 @@ version = "0.9.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" +[[package]] +name = "const-oid" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6ef517f0926dd24a1582492c791b6a4818a4d94e789a334894aa15b0d12f55c" + [[package]] name = "const-random" version = "0.1.18" @@ -2888,9 +2894,9 @@ dependencies = [ [[package]] name = "ctutils" -version = "0.4.2" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d5515a3834141de9eafb9717ad39eea8247b5674e6066c404e8c4b365d2a29e" +checksum = "03bb0e1cc970d482d121d9a1744999169b69a07470b3d644a7894e53fcaf4574" dependencies = [ "cmov", ] @@ -3894,7 +3900,7 @@ dependencies = [ "aws-lc-rs", "base64 0.22.1", "cbc", - "const-oid", + "const-oid 0.9.6", "ctr", "curve25519-dalek", "deno_core", @@ -4306,7 +4312,7 @@ version = "0.7.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e7c1832837b905bbfb5101e07cc24c8deddf52f93225eee6ead5f4d63d53ddcb" dependencies = [ - "const-oid", + "const-oid 0.9.6", "der_derive", "flagset", "pem-rfc7468", @@ -4501,7 +4507,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ "block-buffer 0.10.4", - "const-oid", + "const-oid 0.9.6", "crypto-common 0.1.7", "subtle", ] @@ -4513,6 +4519,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" dependencies = [ "block-buffer 0.12.1", + "const-oid 0.10.2", "crypto-common 0.2.2", "ctutils", ] @@ -5675,7 +5682,7 @@ dependencies = [ "google-cloud-metadata", "google-cloud-token", "hex", - "hmac", + "hmac 0.12.1", "home", "jsonwebtoken 9.3.1", "path-clean", @@ -6042,7 +6049,7 @@ version = "0.12.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7b5f8eb2ad728638ea2c7d47a21db23b7b58a72ed6a38256b8a1849f15fbbdf7" dependencies = [ - "hmac", + "hmac 0.12.1", ] [[package]] @@ -6054,6 +6061,15 @@ dependencies = [ "digest 0.10.7", ] +[[package]] +name = "hmac" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6303bc9732ae41b04cb554b844a762b4115a61bfaa81e3e83050991eeb56863f" +dependencies = [ + "digest 0.11.3", +] + [[package]] name = "home" version = "0.5.12" @@ -7252,9 +7268,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.189" +version = "0.2.190" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" +checksum = "ce5d3ddc6d3fa000eb1536d85e147bfe31aacaba692ed6a876f95cb7c855be78" [[package]] name = "libffi-sys" @@ -7709,6 +7725,16 @@ dependencies = [ "digest 0.10.7", ] +[[package]] +name = "md-5" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69b6441f590336821bb897fb28fc622898ccceb1d6cea3fde5ea86b090c4de98" +dependencies = [ + "cfg-if", + "digest 0.11.3", +] + [[package]] name = "md5" version = "0.6.1" @@ -7830,9 +7856,9 @@ dependencies = [ [[package]] name = "mio" -version = "1.2.3" +version = "1.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b18443e9c262bfe8fa82f51666e2642c53393f7e5c27b3e1aeab922cff5b9d8" +checksum = "1788edb87fdc09c7e26304471e2f5be8cdefb1b6930d6e3985fc02ff53bf86ee" dependencies = [ "libc", "wasi 0.11.1+wasi-snapshot-preview1", @@ -8559,7 +8585,7 @@ dependencies = [ "chrono", "dyn-clone", "ed25519-dalek", - "hmac", + "hmac 0.12.1", "http 1.5.0", "itertools 0.10.5", "log", @@ -8916,7 +8942,7 @@ dependencies = [ "der", "des 0.8.1", "hex", - "hmac", + "hmac 0.12.1", "pkcs12", "pkcs5", "rand 0.9.5", @@ -9080,7 +9106,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8ed6a7761f76e3b9f92dfb0a60a6a6477c61024b775147ff0973a02653abaf2" dependencies = [ "digest 0.10.7", - "hmac", + "hmac 0.12.1", ] [[package]] @@ -9366,7 +9392,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "695b3df3d3cc1015f12d70235e35b6b79befc5fa7a9b95b951eab1dd07c9efc2" dependencies = [ "cms", - "const-oid", + "const-oid 0.9.6", "der", "digest 0.10.7", "spki", @@ -9471,7 +9497,7 @@ dependencies = [ [[package]] name = "postgres-native-tls" version = "0.5.2" -source = "git+https://github.com/MaterializeInc/rust-postgres?rev=78c1222577bb091d69bc22b1bc7ad01c14675abe#78c1222577bb091d69bc22b1bc7ad01c14675abe" +source = "git+https://github.com/windmill-labs/rust-postgres?rev=b3a6443aa1ae90a4f6c4b742103ba80584598f2e#b3a6443aa1ae90a4f6c4b742103ba80584598f2e" dependencies = [ "native-tls", "tokio", @@ -9493,25 +9519,25 @@ dependencies = [ [[package]] name = "postgres-protocol" -version = "0.6.9" -source = "git+https://github.com/MaterializeInc/rust-postgres?rev=78c1222577bb091d69bc22b1bc7ad01c14675abe#78c1222577bb091d69bc22b1bc7ad01c14675abe" +version = "0.6.12" +source = "git+https://github.com/windmill-labs/rust-postgres?rev=b3a6443aa1ae90a4f6c4b742103ba80584598f2e#b3a6443aa1ae90a4f6c4b742103ba80584598f2e" dependencies = [ "base64 0.22.1", "byteorder", "bytes", "fallible-iterator", - "hmac", - "md-5 0.10.6", + "hmac 0.13.0", + "md-5 0.11.0", "memchr", - "rand 0.9.5", - "sha2 0.10.9", + "rand 0.10.3", + "sha2 0.11.0", "stringprep", ] [[package]] name = "postgres-types" version = "0.2.11" -source = "git+https://github.com/MaterializeInc/rust-postgres?rev=78c1222577bb091d69bc22b1bc7ad01c14675abe#78c1222577bb091d69bc22b1bc7ad01c14675abe" +source = "git+https://github.com/windmill-labs/rust-postgres?rev=b3a6443aa1ae90a4f6c4b742103ba80584598f2e#b3a6443aa1ae90a4f6c4b742103ba80584598f2e" dependencies = [ "array-init", "bit-vec 0.6.3", @@ -9537,9 +9563,9 @@ dependencies = [ [[package]] name = "powerfmt" -version = "0.2.0" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" +checksum = "4a6394b9e965e73d0a289ee54f589087e2c676aedf60885baf52c76b771e4958" [[package]] name = "ppv-lite86" @@ -10471,7 +10497,7 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8dd2a808d456c4a54e300a23e9f5a67e122c3024119acbfd73e3bf664491cb2" dependencies = [ - "hmac", + "hmac 0.12.1", "subtle", ] @@ -10569,9 +10595,9 @@ dependencies = [ [[package]] name = "rmcp-macros" -version = "3.5.0" +version = "3.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7e66877136e969d5a2d561ee5841b17717a0619aebd64abb47a2a2f46e34683" +checksum = "a1da498700030856ba63114bceefa30ba21f5e278cb975e5b9de3529fe842b40" dependencies = [ "darling 0.24.1", "proc-macro2", @@ -10634,7 +10660,7 @@ version = "0.9.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b8573f03f5883dcaebdfcf4725caa1ecb9c15b2ef50c43a07b816e06799bb12d" dependencies = [ - "const-oid", + "const-oid 0.9.6", "digest 0.10.7", "num-bigint-dig", "num-integer", @@ -11658,6 +11684,17 @@ dependencies = [ "digest 0.10.7", ] +[[package]] +name = "sha2" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "446ba717509524cb3f22f17ecc096f10f4822d76ab5c0b9822c5f9c284e825f4" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.1", + "digest 0.11.3", +] + [[package]] name = "sharded-slab" version = "0.1.7" @@ -12077,7 +12114,7 @@ dependencies = [ "generic-array", "hex", "hkdf", - "hmac", + "hmac 0.12.1", "itoa", "log", "md-5 0.10.6", @@ -12117,7 +12154,7 @@ dependencies = [ "futures-util", "hex", "hkdf", - "hmac", + "hmac 0.12.1", "home", "itoa", "log", @@ -13380,7 +13417,7 @@ dependencies = [ [[package]] name = "tokio-postgres" version = "0.7.15" -source = "git+https://github.com/MaterializeInc/rust-postgres?rev=78c1222577bb091d69bc22b1bc7ad01c14675abe#78c1222577bb091d69bc22b1bc7ad01c14675abe" +source = "git+https://github.com/windmill-labs/rust-postgres?rev=b3a6443aa1ae90a4f6c4b742103ba80584598f2e#b3a6443aa1ae90a4f6c4b742103ba80584598f2e" dependencies = [ "async-trait", "byteorder", @@ -14029,9 +14066,9 @@ dependencies = [ [[package]] name = "twox-hash" -version = "2.1.4" +version = "2.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5283634e518fe9e82c7b20520bb4bc209009fd16c82077c802f8111ecbb0117a" +checksum = "86a801b3cea342a06d468c8710662aa29e5e05e4f5c0d62f00bbb7f2ad7941c2" [[package]] name = "typed-path" @@ -14158,9 +14195,9 @@ dependencies = [ [[package]] name = "unicase" -version = "2.9.0" +version = "2.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dbc4bc3a9f746d862c45cb89d705aa10f187bb96c76001afab07a0d35ce60142" +checksum = "357cc3acc6a036009fd6c973ed009037c732d60d0b4f6c673e9041497482a28f" [[package]] name = "unicode-bidi" @@ -14379,9 +14416,9 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] name = "uuid" -version = "1.26.1" +version = "1.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ef6dac1e96601b4fb3acccccff2139741fcb757cb9a36089bf5be91cfb285ce" +checksum = "97277d36b9c3ace13e58fa6e753f8b0bbbf302a18dd193240d70f9e29681059a" dependencies = [ "getrandom 0.4.3", "js-sys", @@ -14466,9 +14503,9 @@ dependencies = [ [[package]] name = "want" -version = "0.3.1" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +checksum = "ec4cdd0dd910afe868b7ef477227d8d538b46b3075031afee8a9f2acb0a2ed0b" dependencies = [ "try-lock", ] @@ -14792,7 +14829,7 @@ dependencies = [ [[package]] name = "windmill" -version = "1.821.0" +version = "1.825.0" dependencies = [ "anyhow", "async-nats", @@ -14808,7 +14845,7 @@ dependencies = [ "gethostname", "git-version", "hex", - "hmac", + "hmac 0.12.1", "jsonwebtoken 10.4.0", "lazy_static", "once_cell", @@ -14881,7 +14918,7 @@ dependencies = [ [[package]] name = "windmill-ai" -version = "1.821.0" +version = "1.825.0" dependencies = [ "async-stream", "async-trait", @@ -14915,7 +14952,7 @@ dependencies = [ [[package]] name = "windmill-alerting" -version = "1.821.0" +version = "1.825.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14928,7 +14965,7 @@ dependencies = [ [[package]] name = "windmill-api" -version = "1.821.0" +version = "1.825.0" dependencies = [ "anyhow", "argon2", @@ -14956,7 +14993,7 @@ dependencies = [ "futures", "git-version", "hex", - "hmac", + "hmac 0.12.1", "http 1.5.0", "hyper 1.11.1", "indexmap 2.14.2", @@ -15068,7 +15105,7 @@ dependencies = [ [[package]] name = "windmill-api-agent-workers" -version = "1.821.0" +version = "1.825.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15091,7 +15128,7 @@ dependencies = [ [[package]] name = "windmill-api-assets" -version = "1.821.0" +version = "1.825.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15108,7 +15145,7 @@ dependencies = [ [[package]] name = "windmill-api-auth" -version = "1.821.0" +version = "1.825.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -15134,7 +15171,7 @@ dependencies = [ [[package]] name = "windmill-api-client" -version = "1.821.0" +version = "1.825.0" dependencies = [ "reqwest 0.12.28", "serde", @@ -15144,7 +15181,7 @@ dependencies = [ [[package]] name = "windmill-api-configs" -version = "1.821.0" +version = "1.825.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15161,7 +15198,7 @@ dependencies = [ [[package]] name = "windmill-api-debug" -version = "1.821.0" +version = "1.825.0" dependencies = [ "axum 0.8.9", "base64 0.22.1", @@ -15183,7 +15220,7 @@ dependencies = [ [[package]] name = "windmill-api-embeddings" -version = "1.821.0" +version = "1.825.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -15206,7 +15243,7 @@ dependencies = [ [[package]] name = "windmill-api-flow-conversations" -version = "1.821.0" +version = "1.825.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15222,7 +15259,7 @@ dependencies = [ [[package]] name = "windmill-api-flows" -version = "1.821.0" +version = "1.825.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15244,7 +15281,7 @@ dependencies = [ [[package]] name = "windmill-api-groups" -version = "1.821.0" +version = "1.825.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15266,7 +15303,7 @@ dependencies = [ [[package]] name = "windmill-api-inputs" -version = "1.821.0" +version = "1.825.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15280,7 +15317,7 @@ dependencies = [ [[package]] name = "windmill-api-integration-tests" -version = "1.821.0" +version = "1.825.0" dependencies = [ "anyhow", "async-nats", @@ -15291,7 +15328,7 @@ dependencies = [ "base64 0.22.1", "futures", "hex", - "hmac", + "hmac 0.12.1", "rand 0.9.5", "rdkafka", "reqwest 0.13.5", @@ -15316,7 +15353,7 @@ dependencies = [ [[package]] name = "windmill-api-jobs" -version = "1.821.0" +version = "1.825.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -15341,7 +15378,7 @@ dependencies = [ [[package]] name = "windmill-api-npm-proxy" -version = "1.821.0" +version = "1.825.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15369,7 +15406,7 @@ dependencies = [ [[package]] name = "windmill-api-openapi" -version = "1.821.0" +version = "1.825.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -15391,7 +15428,7 @@ dependencies = [ [[package]] name = "windmill-api-schedule" -version = "1.821.0" +version = "1.825.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15411,7 +15448,7 @@ dependencies = [ [[package]] name = "windmill-api-scripts" -version = "1.821.0" +version = "1.825.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -15453,7 +15490,7 @@ dependencies = [ [[package]] name = "windmill-api-settings" -version = "1.821.0" +version = "1.825.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -15482,7 +15519,7 @@ dependencies = [ [[package]] name = "windmill-api-sse" -version = "1.821.0" +version = "1.825.0" dependencies = [ "lazy_static", "serde", @@ -15494,7 +15531,7 @@ dependencies = [ [[package]] name = "windmill-api-users" -version = "1.821.0" +version = "1.825.0" dependencies = [ "argon2", "axum 0.8.9", @@ -15518,7 +15555,7 @@ dependencies = [ [[package]] name = "windmill-api-workers" -version = "1.821.0" +version = "1.825.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15532,7 +15569,7 @@ dependencies = [ [[package]] name = "windmill-api-workspaces" -version = "1.821.0" +version = "1.825.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15571,7 +15608,7 @@ dependencies = [ [[package]] name = "windmill-audit" -version = "1.821.0" +version = "1.825.0" dependencies = [ "chrono", "lazy_static", @@ -15585,7 +15622,7 @@ dependencies = [ [[package]] name = "windmill-autoscaling" -version = "1.821.0" +version = "1.825.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -15604,7 +15641,7 @@ dependencies = [ [[package]] name = "windmill-common" -version = "1.821.0" +version = "1.825.0" dependencies = [ "aes-gcm", "aho-corasick", @@ -15640,7 +15677,7 @@ dependencies = [ "git-version", "globset", "hex", - "hmac", + "hmac 0.12.1", "hyper 1.11.1", "indexmap 2.14.2", "itertools 0.14.0", @@ -15674,6 +15711,7 @@ dependencies = [ "rsa", "rustls 0.23.35", "rustls-native-certs 0.8.4", + "ryu-js", "schemars 0.8.22", "semver 1.0.28", "serde", @@ -15717,7 +15755,7 @@ dependencies = [ [[package]] name = "windmill-dep-map" -version = "1.821.0" +version = "1.825.0" dependencies = [ "chrono", "futures", @@ -15737,7 +15775,7 @@ dependencies = [ [[package]] name = "windmill-git-sync" -version = "1.821.0" +version = "1.825.0" dependencies = [ "regex", "serde", @@ -15754,7 +15792,7 @@ dependencies = [ [[package]] name = "windmill-indexer" -version = "1.821.0" +version = "1.825.0" dependencies = [ "anyhow", "astral-tokio-tar", @@ -15781,7 +15819,7 @@ dependencies = [ [[package]] name = "windmill-jseval" -version = "1.821.0" +version = "1.825.0" dependencies = [ "anyhow", "futures", @@ -15798,7 +15836,7 @@ dependencies = [ [[package]] name = "windmill-macros" -version = "1.821.0" +version = "1.825.0" dependencies = [ "itertools 0.14.0", "lazy_static", @@ -15814,7 +15852,7 @@ dependencies = [ [[package]] name = "windmill-mcp" -version = "1.821.0" +version = "1.825.0" dependencies = [ "anyhow", "async-trait", @@ -15835,7 +15873,7 @@ dependencies = [ [[package]] name = "windmill-native-triggers" -version = "1.821.0" +version = "1.825.0" dependencies = [ "anyhow", "async-trait", @@ -15843,7 +15881,7 @@ dependencies = [ "backon", "base64 0.22.1", "chrono", - "hmac", + "hmac 0.12.1", "http 1.5.0", "itertools 0.14.0", "lazy_static", @@ -15866,7 +15904,7 @@ dependencies = [ [[package]] name = "windmill-oauth" -version = "1.821.0" +version = "1.825.0" dependencies = [ "anyhow", "arc-swap", @@ -15875,7 +15913,7 @@ dependencies = [ "base64 0.22.1", "chrono", "hex", - "hmac", + "hmac 0.12.1", "itertools 0.14.0", "lazy_static", "reqwest 0.12.28", @@ -15891,7 +15929,7 @@ dependencies = [ [[package]] name = "windmill-object-store" -version = "1.821.0" +version = "1.825.0" dependencies = [ "anyhow", "async-stream", @@ -15927,7 +15965,7 @@ dependencies = [ [[package]] name = "windmill-operator" -version = "1.821.0" +version = "1.825.0" dependencies = [ "anyhow", "futures", @@ -15945,7 +15983,7 @@ dependencies = [ [[package]] name = "windmill-parser" -version = "1.821.0" +version = "1.825.0" dependencies = [ "convert_case 0.6.0", "serde", @@ -15954,7 +15992,7 @@ dependencies = [ [[package]] name = "windmill-parser-bash" -version = "1.821.0" +version = "1.825.0" dependencies = [ "anyhow", "lazy_static", @@ -15966,7 +16004,7 @@ dependencies = [ [[package]] name = "windmill-parser-csharp" -version = "1.821.0" +version = "1.825.0" dependencies = [ "anyhow", "serde_json", @@ -15978,7 +16016,7 @@ dependencies = [ [[package]] name = "windmill-parser-go" -version = "1.821.0" +version = "1.825.0" dependencies = [ "anyhow", "gosyn", @@ -15990,7 +16028,7 @@ dependencies = [ [[package]] name = "windmill-parser-graphql" -version = "1.821.0" +version = "1.825.0" dependencies = [ "anyhow", "lazy_static", @@ -16002,7 +16040,7 @@ dependencies = [ [[package]] name = "windmill-parser-java" -version = "1.821.0" +version = "1.825.0" dependencies = [ "anyhow", "serde_json", @@ -16014,7 +16052,7 @@ dependencies = [ [[package]] name = "windmill-parser-nu" -version = "1.821.0" +version = "1.825.0" dependencies = [ "anyhow", "nu-parser", @@ -16025,7 +16063,7 @@ dependencies = [ [[package]] name = "windmill-parser-php" -version = "1.821.0" +version = "1.825.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -16036,7 +16074,7 @@ dependencies = [ [[package]] name = "windmill-parser-py" -version = "1.821.0" +version = "1.825.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -16048,7 +16086,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-asset" -version = "1.821.0" +version = "1.825.0" dependencies = [ "anyhow", "rustpython-ast", @@ -16059,7 +16097,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-imports" -version = "1.821.0" +version = "1.825.0" dependencies = [ "anyhow", "async-recursion", @@ -16081,7 +16119,7 @@ dependencies = [ [[package]] name = "windmill-parser-r" -version = "1.821.0" +version = "1.825.0" dependencies = [ "anyhow", "serde_json", @@ -16093,7 +16131,7 @@ dependencies = [ [[package]] name = "windmill-parser-ruby" -version = "1.821.0" +version = "1.825.0" dependencies = [ "anyhow", "lazy_static", @@ -16107,7 +16145,7 @@ dependencies = [ [[package]] name = "windmill-parser-rust" -version = "1.821.0" +version = "1.825.0" dependencies = [ "anyhow", "convert_case 0.6.0", @@ -16124,7 +16162,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql" -version = "1.821.0" +version = "1.825.0" dependencies = [ "anyhow", "lazy_static", @@ -16137,7 +16175,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql-asset" -version = "1.821.0" +version = "1.825.0" dependencies = [ "anyhow", "serde", @@ -16149,7 +16187,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts" -version = "1.821.0" +version = "1.825.0" dependencies = [ "anyhow", "lazy_static", @@ -16167,7 +16205,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts-asset" -version = "1.821.0" +version = "1.825.0" dependencies = [ "anyhow", "serde-wasm-bindgen", @@ -16183,7 +16221,7 @@ dependencies = [ [[package]] name = "windmill-parser-wac" -version = "1.821.0" +version = "1.825.0" dependencies = [ "anyhow", "rustpython-ast", @@ -16199,7 +16237,7 @@ dependencies = [ [[package]] name = "windmill-parser-yaml" -version = "1.821.0" +version = "1.825.0" dependencies = [ "anyhow", "lazy_static", @@ -16213,7 +16251,7 @@ dependencies = [ [[package]] name = "windmill-queue" -version = "1.821.0" +version = "1.825.0" dependencies = [ "anyhow", "async-recursion", @@ -16226,7 +16264,7 @@ dependencies = [ "futures", "futures-core", "hex", - "hmac", + "hmac 0.12.1", "itertools 0.14.0", "lazy_static", "once_cell", @@ -16252,7 +16290,7 @@ dependencies = [ [[package]] name = "windmill-runtime-nativets" -version = "1.821.0" +version = "1.825.0" dependencies = [ "anyhow", "const_format", @@ -16292,7 +16330,7 @@ dependencies = [ [[package]] name = "windmill-sql-datatype-parser-wasm" -version = "1.821.0" +version = "1.825.0" dependencies = [ "getrandom 0.3.4", "wasm-bindgen", @@ -16303,7 +16341,7 @@ dependencies = [ [[package]] name = "windmill-store" -version = "1.821.0" +version = "1.825.0" dependencies = [ "anyhow", "async-recursion", @@ -16338,7 +16376,7 @@ dependencies = [ [[package]] name = "windmill-test-utils" -version = "1.821.0" +version = "1.825.0" dependencies = [ "anyhow", "async-trait", @@ -16362,7 +16400,7 @@ dependencies = [ [[package]] name = "windmill-trigger" -version = "1.821.0" +version = "1.825.0" dependencies = [ "anyhow", "async-trait", @@ -16395,7 +16433,7 @@ dependencies = [ [[package]] name = "windmill-trigger-amqp" -version = "1.821.0" +version = "1.825.0" dependencies = [ "anyhow", "async-trait", @@ -16422,7 +16460,7 @@ dependencies = [ [[package]] name = "windmill-trigger-azure" -version = "1.821.0" +version = "1.825.0" dependencies = [ "anyhow", "async-trait", @@ -16455,7 +16493,7 @@ dependencies = [ [[package]] name = "windmill-trigger-email" -version = "1.821.0" +version = "1.825.0" dependencies = [ "anyhow", "async-trait", @@ -16475,7 +16513,7 @@ dependencies = [ [[package]] name = "windmill-trigger-gcp" -version = "1.821.0" +version = "1.825.0" dependencies = [ "anyhow", "async-trait", @@ -16509,7 +16547,7 @@ dependencies = [ [[package]] name = "windmill-trigger-http" -version = "1.821.0" +version = "1.825.0" dependencies = [ "anyhow", "async-trait", @@ -16519,7 +16557,7 @@ dependencies = [ "constant_time_eq 0.3.1", "futures", "hex", - "hmac", + "hmac 0.12.1", "http 1.5.0", "hyper 1.11.1", "itertools 0.14.0", @@ -16546,7 +16584,7 @@ dependencies = [ [[package]] name = "windmill-trigger-kafka" -version = "1.821.0" +version = "1.825.0" dependencies = [ "anyhow", "async-trait", @@ -16569,7 +16607,7 @@ dependencies = [ [[package]] name = "windmill-trigger-mqtt" -version = "1.821.0" +version = "1.825.0" dependencies = [ "anyhow", "async-trait", @@ -16593,7 +16631,7 @@ dependencies = [ [[package]] name = "windmill-trigger-nats" -version = "1.821.0" +version = "1.825.0" dependencies = [ "anyhow", "async-nats", @@ -16617,7 +16655,7 @@ dependencies = [ [[package]] name = "windmill-trigger-postgres" -version = "1.821.0" +version = "1.825.0" dependencies = [ "anyhow", "async-trait", @@ -16652,7 +16690,7 @@ dependencies = [ [[package]] name = "windmill-trigger-sqs" -version = "1.821.0" +version = "1.825.0" dependencies = [ "anyhow", "async-trait", @@ -16680,7 +16718,7 @@ dependencies = [ [[package]] name = "windmill-trigger-websocket" -version = "1.821.0" +version = "1.825.0" dependencies = [ "anyhow", "async-trait", @@ -16705,7 +16743,7 @@ dependencies = [ [[package]] name = "windmill-types" -version = "1.821.0" +version = "1.825.0" dependencies = [ "anyhow", "bitflags 2.13.2", @@ -16724,7 +16762,7 @@ dependencies = [ [[package]] name = "windmill-worker" -version = "1.821.0" +version = "1.825.0" dependencies = [ "anyhow", "async-once-cell", @@ -16750,7 +16788,7 @@ dependencies = [ "gcp_auth", "git-version", "hex", - "hmac", + "hmac 0.12.1", "hudsucker", "hyper-http-proxy", "hyper-rustls 0.27.10", @@ -16773,6 +16811,7 @@ dependencies = [ "pem", "pep440_rs", "postgres-native-tls 0.5.3", + "postgres-protocol", "process-wrap", "prometheus", "prost", @@ -16842,7 +16881,7 @@ dependencies = [ [[package]] name = "windmill-worker-volumes" -version = "1.821.0" +version = "1.825.0" dependencies = [ "bytes", "futures", @@ -17437,7 +17476,7 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1301e935010a701ae5f8655edc0ad17c44bad3ac5ce8c39185f75453b720ae94" dependencies = [ - "const-oid", + "const-oid 0.9.6", "der", "spki", ] @@ -17551,18 +17590,18 @@ dependencies = [ [[package]] name = "zerocopy" -version = "0.8.59" +version = "0.8.60" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6df92bf3d9227be3d53173901ddbffac2babc27ae50f397776ffd6dc33f800cb" +checksum = "6400fc4bb7426f6faac3fb07566122bb18622e3022bd2c36426a92ed489b1dbe" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.59" +version = "0.8.60" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac4f328cf2f05d084e496c3e9c3f33ed0a183656a16e1fcec4d464d8373aec82" +checksum = "99c27cc7525bad4df59a45e073c45b2c2aa820efb588234cc23a547ec69a34db" dependencies = [ "proc-macro2", "quote", diff --git a/backend/Cargo.toml b/backend/Cargo.toml index d2bec22817..4563ed5b03 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "windmill" -version = "1.821.0" +version = "1.825.0" authors.workspace = true edition.workspace = true @@ -88,7 +88,7 @@ members = [ exclude = ["./windmill-duckdb-ffi-internal", "./parsers/windmill-parser-wasm"] [workspace.package] -version = "1.821.0" +version = "1.825.0" authors = ["Ruben Fiszel "] edition = "2021" @@ -249,36 +249,22 @@ sqlx-sqlite = { git = "https://github.com/windmill-labs/sqlx", rev = "6bdaee94fa object_store = { git = "https://github.com/apache/arrow-rs-object-store", rev = "36752c975d4f29e20b57c91f81a10872dcd48ae7" } # Use tiberius main branch for libgssapi 0.8.1 fix (https://github.com/prisma/tiberius/issues/343) tiberius = { git = "https://github.com/prisma/tiberius", rev = "59db57960a14b422fb3a1309aa4aa47880896ff8" } -# Pin tokio-postgres / postgres-types / postgres-protocol to the -# MaterializeInc fork. windmill-trigger-postgres already pulled this -# fork in transitively for the postgres-replication crate -# (CopyBothDuplex, LogicalReplicationStream, TupleData with binary -# tuple support) which upstream rust-postgres has declined to merge -# since 2021 (PR #752 → #778, both still unmerged). +# Pin tokio-postgres / postgres-types / postgres-protocol to +# windmill-labs/rust-postgres, branch `windmill-describe-first`: the +# MaterializeInc fork (postgres-replication, which upstream has not merged) +# plus `Client::describe_typed`, which pg_executor needs before +# query_typed_raw so result columns of types the client has not seen yet +# (custom enums / domains, citext, postgis) don't deadlock a large result, +# and `Client::transaction_status`, which keeps a connection left inside a +# transaction out of the executor's cache. Each pinned commit carries a `windmill-*` tag so it stays reachable when the +# branch is rebased onto MaterializeInc master. # -# MI also carries a mitigation for the -# Client::query_typed_raw / Client::prepare deadlock on result columns -# whose Oid the client doesn't know about yet (citext, custom enums / -# domains, postgis): MI's 2025-12-11 PR #33 resized the per-request -# response channel from mpsc::channel(1) → mpsc::channel(1024). -# bounded(1024) is sufficient for any realistic typeinfo deferral -# (need ~2-3 batches) but leaves a theoretical failure mode at -# >~64 MB results with a custom-Oid column. The strict-correct fix is -# mpsc::unbounded(); a follow-up PR to MI is open proposing that. -# -# The [patch.crates-io] entries below force windmill-worker's -# pg_executor (which imports `tokio_postgres::` directly from -# crates.io) onto the same fork as windmill-trigger-postgres, so the -# deadlock mitigation reaches both consumers. -# -# Upstream deadlock PRs (open, not on the critical path now that MI -# is mitigated): -# https://github.com/rust-postgres/rust-postgres/pull/1348 -# https://github.com/rust-postgres/rust-postgres/pull/1349 -# Reproducer: https://github.com/rubenfiszel/tokio-postgres-deadlock-repro -tokio-postgres = { git = "https://github.com/MaterializeInc/rust-postgres", rev = "78c1222577bb091d69bc22b1bc7ad01c14675abe" } -postgres-types = { git = "https://github.com/MaterializeInc/rust-postgres", rev = "78c1222577bb091d69bc22b1bc7ad01c14675abe" } -postgres-protocol = { git = "https://github.com/MaterializeInc/rust-postgres", rev = "78c1222577bb091d69bc22b1bc7ad01c14675abe" } +# The [patch.crates-io] entries force windmill-worker's pg_executor (which +# imports `tokio_postgres::` from crates.io) onto the same fork as +# windmill-trigger-postgres. +tokio-postgres = { git = "https://github.com/windmill-labs/rust-postgres", rev = "b3a6443aa1ae90a4f6c4b742103ba80584598f2e" } +postgres-types = { git = "https://github.com/windmill-labs/rust-postgres", rev = "b3a6443aa1ae90a4f6c4b742103ba80584598f2e" } +postgres-protocol = { git = "https://github.com/windmill-labs/rust-postgres", rev = "b3a6443aa1ae90a4f6c4b742103ba80584598f2e" } [dependencies] anyhow.workspace = true @@ -473,7 +459,7 @@ tower-cookies = "^0.11" serde = "^1" # 1.0.151 introduced RawValue::from_string_unchecked, which the SQL executors use # to avoid re-parsing every collected row. -serde_json = { version = "^1.0.151", features = ["preserve_order", "raw_value"] } +serde_json = { version = "^1.0.151", features = ["preserve_order", "raw_value", "float_roundtrip"] } serde_yml = "0.0.12" uuid = { version = "^1", features = ["serde", "v4", "js"] } thiserror = "^2" @@ -586,6 +572,7 @@ base64 = "^0.22.1" base32 = "^0" hmac = "0.12.1" sha2 = "0.10.6" +ryu-js = "1.0.3" md-5 = "0.10.6" sha1 = "0.10.6" sqlx = { version = "0.8.0", features = [ @@ -630,8 +617,8 @@ convert_case = "0.6.0" getrandom = "0.2" tokio-postgres = {version = "^0.7", features = ["array-impls", "with-serde_json-1", "with-chrono-0_4", "with-uuid-1", "with-bit-vec-0_6"]} postgres-protocol = "0.6" -rust-postgres = { package = "tokio-postgres", git = "https://github.com/MaterializeInc/rust-postgres", rev = "78c1222577bb091d69bc22b1bc7ad01c14675abe"} -rust-postgres-native-tls = { package = "postgres-native-tls", git = "https://github.com/MaterializeInc/rust-postgres", features = ["runtime"], rev = "78c1222577bb091d69bc22b1bc7ad01c14675abe" } +rust-postgres = { package = "tokio-postgres", git = "https://github.com/windmill-labs/rust-postgres", rev = "b3a6443aa1ae90a4f6c4b742103ba80584598f2e"} +rust-postgres-native-tls = { package = "postgres-native-tls", git = "https://github.com/windmill-labs/rust-postgres", features = ["runtime"], rev = "b3a6443aa1ae90a4f6c4b742103ba80584598f2e" } bit-vec = "=0.6.3" mappable-rc = "^0" mysql_async = { version = "*", default-features = false, features = ["minimal", "default", "native-tls-tls", "rust_decimal"]} diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index c2c253e8f0..7f8d64aa9d 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -ff4d04f17721d84fb9a1f655af50ff7e4e7dcbc7 +065c2c64c3904d5c0b2766b424210d34f9e54d2d \ No newline at end of file diff --git a/backend/migrations/20260929064555_flow_conversation_message_turn_job_id.down.sql b/backend/migrations/20260929064555_flow_conversation_message_turn_job_id.down.sql new file mode 100644 index 0000000000..5906eda25d --- /dev/null +++ b/backend/migrations/20260929064555_flow_conversation_message_turn_job_id.down.sql @@ -0,0 +1 @@ +ALTER TABLE flow_conversation_message DROP COLUMN turn_job_id; diff --git a/backend/migrations/20260929064555_flow_conversation_message_turn_job_id.up.sql b/backend/migrations/20260929064555_flow_conversation_message_turn_job_id.up.sql new file mode 100644 index 0000000000..db9427692b --- /dev/null +++ b/backend/migrations/20260929064555_flow_conversation_message_turn_job_id.up.sql @@ -0,0 +1,23 @@ +-- The flow run a row's turn belongs to: the job the user message started. Retention collects +-- rows by it, so a turn goes as one unit. `job_id` names the step or tool job that wrote the +-- row, which completes, and so expires, before the run does. +ALTER TABLE flow_conversation_message ADD COLUMN turn_job_id UUID; + +-- Turns can overlap in one conversation, so a row's turn is the user row its job descends +-- from. Transcript order only decides for rows whose job is already gone or that have none. +UPDATE flow_conversation_message m + SET turn_job_id = COALESCE( + (SELECT u.job_id FROM v2_job j + JOIN flow_conversation_message u + ON u.conversation_id = m.conversation_id + AND u.message_type = 'user' + AND u.job_id IN (j.id, j.parent_job, j.root_job, j.flow_innermost_root_job) + WHERE j.id = m.job_id + LIMIT 1), + (SELECT u.job_id FROM flow_conversation_message u + WHERE u.conversation_id = m.conversation_id + AND u.message_type = 'user' + AND u.created_seq <= m.created_seq + ORDER BY u.created_seq DESC + LIMIT 1) + ); diff --git a/backend/migrations/20261002124918_job_token_scopes.down.sql b/backend/migrations/20261002124918_job_token_scopes.down.sql new file mode 100644 index 0000000000..be2c13b3ff --- /dev/null +++ b/backend/migrations/20261002124918_job_token_scopes.down.sql @@ -0,0 +1,3 @@ +ALTER TABLE job_perms DROP COLUMN IF EXISTS job_token_scopes; +ALTER TABLE flow DROP COLUMN IF EXISTS job_token_scopes; +ALTER TABLE script DROP COLUMN IF EXISTS job_token_scopes; diff --git a/backend/migrations/20261002124918_job_token_scopes.up.sql b/backend/migrations/20261002124918_job_token_scopes.up.sql new file mode 100644 index 0000000000..6b24325378 --- /dev/null +++ b/backend/migrations/20261002124918_job_token_scopes.up.sql @@ -0,0 +1,3 @@ +ALTER TABLE script ADD COLUMN IF NOT EXISTS job_token_scopes text[]; +ALTER TABLE flow ADD COLUMN IF NOT EXISTS job_token_scopes text[]; +ALTER TABLE job_perms ADD COLUMN IF NOT EXISTS job_token_scopes text[]; diff --git a/backend/oauth_connect.json b/backend/oauth_connect.json index fde54c39e2..3c4de3b082 100644 --- a/backend/oauth_connect.json +++ b/backend/oauth_connect.json @@ -45,12 +45,39 @@ "files:read" ], "scope_options": [ + { "value": "chat:write", "label": "Send messages as the app" }, + { "value": "chat:write.public", "label": "Send messages to public channels the app is not in" }, + { "value": "channels:join", "label": "Join public channels" }, + { "value": "files:write", "label": "Upload, edit and delete files" }, + { "value": "users:read", "label": "View people in the workspace" }, + { "value": "users:read.email", "label": "View people's email addresses" }, + { "value": "channels:read", "label": "View basic information about public channels" }, + { "value": "channels:history", "label": "Read messages in public channels the app is in" }, { "value": "reactions:write", "label": "Add emoji reactions" }, { "value": "channels:manage", "label": "Create, archive and manage public channels, invite members, set topics" }, { "value": "groups:write", "label": "Create and manage private channels the app is in" }, - { "value": "usergroups:write", "label": "Edit user groups (workspace settings often block bots)" } + { "value": "usergroups:write", "label": "Edit user groups (workspace settings often block bots)" }, + { "value": "files:read", "label": "Read and download files shared where the app is a member" }, + { "value": "groups:read", "label": "List private channels the app is in" }, + { "value": "im:read", "label": "List the app's direct messages" }, + { "value": "mpim:read", "label": "List the app's group direct messages" }, + { "value": "usergroups:read", "label": "List user groups" }, + { "value": "emoji:read", "label": "List custom emojis" } ], "user_scope_options": [ + { "value": "channels:history", "label": "Read messages in public channels" }, + { "value": "groups:history", "label": "Read messages in your private channels" }, + { "value": "im:history", "label": "Read your direct messages" }, + { "value": "mpim:history", "label": "Read your group direct messages" }, + { "value": "search:read", "label": "Search messages and files" }, + { "value": "users:read", "label": "View people in the workspace" }, + { "value": "chat:write", "label": "Send messages as you" }, + { "value": "users:read.email", "label": "View people's email addresses" }, + { "value": "channels:read", "label": "View basic information about public channels" }, + { "value": "groups:read", "label": "View basic information about your private channels" }, + { "value": "im:read", "label": "View basic information about your direct messages" }, + { "value": "mpim:read", "label": "View basic information about your group direct messages" }, + { "value": "files:read", "label": "View files shared in your conversations" }, { "value": "reactions:write", "label": "Add emoji reactions" }, { "value": "channels:write", "label": "Create, archive and manage public channels" }, { "value": "channels:write.invites", "label": "Invite members to public channels" }, @@ -58,7 +85,10 @@ { "value": "groups:write", "label": "Create and manage private channels" }, { "value": "users.profile:write", "label": "Set your status and edit your profile" }, { "value": "reminders:write", "label": "Create reminders for yourself" }, - { "value": "usergroups:write", "label": "Edit user groups" } + { "value": "usergroups:write", "label": "Edit user groups" }, + { "value": "files:write", "label": "Upload and delete files as yourself" }, + { "value": "usergroups:read", "label": "List user groups" }, + { "value": "emoji:read", "label": "List custom emojis" } ], "token_response_path": "authed_user" }, @@ -73,7 +103,8 @@ "scopes": ["https://www.googleapis.com/auth/spreadsheets"], "scope_options": [ { "value": "https://www.googleapis.com/auth/spreadsheets", "label": "Read, edit, create and delete spreadsheets" }, - { "value": "https://www.googleapis.com/auth/spreadsheets.readonly", "label": "Read spreadsheets" } + { "value": "https://www.googleapis.com/auth/spreadsheets.readonly", "label": "Read spreadsheets" }, + { "value": "https://www.googleapis.com/auth/drive.metadata.readonly", "label": "List and search your spreadsheets in Drive" } ], "extra_params": { "access_type": "offline", @@ -150,7 +181,9 @@ "scope_options": [ { "value": "https://www.googleapis.com/auth/documents", "label": "Read, edit, create and delete documents" }, { "value": "https://www.googleapis.com/auth/documents.readonly", "label": "Read documents" }, - { "value": "https://www.googleapis.com/auth/drive.file", "label": "Only files created or opened with this connection" } + { "value": "https://www.googleapis.com/auth/drive.file", "label": "Only files created or opened with this connection" }, + { "value": "https://www.googleapis.com/auth/drive.readonly", "label": "Read, export and search all Drive files and comments" }, + { "value": "https://www.googleapis.com/auth/drive", "label": "Read, edit, create and delete all Drive files" } ], "extra_params": { "access_type": "offline", @@ -205,7 +238,8 @@ { "value": "https://www.googleapis.com/auth/admin.directory.group.readonly", "label": "Read groups" }, { "value": "https://www.googleapis.com/auth/admin.directory.orgunit", "label": "Read and manage organizational units" }, { "value": "https://www.googleapis.com/auth/admin.directory.orgunit.readonly", "label": "Read organizational units" }, - { "value": "https://www.googleapis.com/auth/admin.directory.user.security", "label": "Manage user security settings" } + { "value": "https://www.googleapis.com/auth/admin.directory.user.security", "label": "Manage user security settings" }, + { "value": "https://www.googleapis.com/auth/admin.reports.audit.readonly", "label": "Read audit logs (login, admin, Drive, token activity)" } ], "extra_params": { "access_type": "offline", @@ -220,6 +254,14 @@ "type": "web_server" } }, + "notion": { + "auth_url": "https://api.notion.com/v1/oauth/authorize", + "token_url": "https://api.notion.com/v1/oauth/token", + "scopes": [], + "extra_params": { + "owner": "user" + } + }, "linkedin": { "auth_url": "https://www.linkedin.com/oauth/v2/authorization", "token_url": "https://www.linkedin.com/oauth/v2/accessToken", diff --git a/backend/parsers/windmill-parser-wasm/Cargo.lock b/backend/parsers/windmill-parser-wasm/Cargo.lock index 62c0cc5a8d..e5adbb3daf 100644 --- a/backend/parsers/windmill-parser-wasm/Cargo.lock +++ b/backend/parsers/windmill-parser-wasm/Cargo.lock @@ -6216,7 +6216,7 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "windmill-common" -version = "1.821.0" +version = "1.825.0" dependencies = [ "aho-corasick", "anyhow", @@ -6302,7 +6302,7 @@ dependencies = [ [[package]] name = "windmill-macros" -version = "1.821.0" +version = "1.825.0" dependencies = [ "proc-macro2", "quote", @@ -6314,7 +6314,7 @@ dependencies = [ [[package]] name = "windmill-parser" -version = "1.821.0" +version = "1.825.0" dependencies = [ "convert_case", "serde", @@ -6323,7 +6323,7 @@ dependencies = [ [[package]] name = "windmill-parser-bash" -version = "1.821.0" +version = "1.825.0" dependencies = [ "anyhow", "lazy_static", @@ -6335,7 +6335,7 @@ dependencies = [ [[package]] name = "windmill-parser-csharp" -version = "1.821.0" +version = "1.825.0" dependencies = [ "anyhow", "serde_json", @@ -6347,7 +6347,7 @@ dependencies = [ [[package]] name = "windmill-parser-go" -version = "1.821.0" +version = "1.825.0" dependencies = [ "anyhow", "gosyn", @@ -6359,7 +6359,7 @@ dependencies = [ [[package]] name = "windmill-parser-graphql" -version = "1.821.0" +version = "1.825.0" dependencies = [ "anyhow", "lazy_static", @@ -6371,7 +6371,7 @@ dependencies = [ [[package]] name = "windmill-parser-java" -version = "1.821.0" +version = "1.825.0" dependencies = [ "anyhow", "serde_json", @@ -6383,7 +6383,7 @@ dependencies = [ [[package]] name = "windmill-parser-nu" -version = "1.821.0" +version = "1.825.0" dependencies = [ "anyhow", "nu-parser", @@ -6394,7 +6394,7 @@ dependencies = [ [[package]] name = "windmill-parser-php" -version = "1.821.0" +version = "1.825.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -6405,7 +6405,7 @@ dependencies = [ [[package]] name = "windmill-parser-py" -version = "1.821.0" +version = "1.825.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -6417,7 +6417,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-asset" -version = "1.821.0" +version = "1.825.0" dependencies = [ "anyhow", "rustpython-ast", @@ -6428,7 +6428,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-imports" -version = "1.821.0" +version = "1.825.0" dependencies = [ "anyhow", "async-recursion", @@ -6450,7 +6450,7 @@ dependencies = [ [[package]] name = "windmill-parser-r" -version = "1.821.0" +version = "1.825.0" dependencies = [ "anyhow", "serde_json", @@ -6462,7 +6462,7 @@ dependencies = [ [[package]] name = "windmill-parser-ruby" -version = "1.821.0" +version = "1.825.0" dependencies = [ "anyhow", "lazy_static", @@ -6476,7 +6476,7 @@ dependencies = [ [[package]] name = "windmill-parser-rust" -version = "1.821.0" +version = "1.825.0" dependencies = [ "anyhow", "convert_case", @@ -6493,7 +6493,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql" -version = "1.821.0" +version = "1.825.0" dependencies = [ "anyhow", "lazy_static", @@ -6506,7 +6506,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql-asset" -version = "1.821.0" +version = "1.825.0" dependencies = [ "anyhow", "serde", @@ -6518,7 +6518,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts" -version = "1.821.0" +version = "1.825.0" dependencies = [ "anyhow", "lazy_static", @@ -6536,7 +6536,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts-asset" -version = "1.821.0" +version = "1.825.0" dependencies = [ "anyhow", "serde-wasm-bindgen", @@ -6552,7 +6552,7 @@ dependencies = [ [[package]] name = "windmill-parser-wac" -version = "1.821.0" +version = "1.825.0" dependencies = [ "anyhow", "rustpython-ast", @@ -6568,7 +6568,7 @@ dependencies = [ [[package]] name = "windmill-parser-wasm" -version = "1.821.0" +version = "1.825.0" dependencies = [ "anyhow", "getrandom 0.2.17", @@ -6600,7 +6600,7 @@ dependencies = [ [[package]] name = "windmill-parser-yaml" -version = "1.821.0" +version = "1.825.0" dependencies = [ "anyhow", "lazy_static", @@ -6614,7 +6614,7 @@ dependencies = [ [[package]] name = "windmill-types" -version = "1.821.0" +version = "1.825.0" dependencies = [ "anyhow", "bitflags", diff --git a/backend/parsers/windmill-parser-wasm/Cargo.toml b/backend/parsers/windmill-parser-wasm/Cargo.toml index 387a98aa1f..7c11d27605 100644 --- a/backend/parsers/windmill-parser-wasm/Cargo.toml +++ b/backend/parsers/windmill-parser-wasm/Cargo.toml @@ -12,7 +12,7 @@ resolver = "2" members = ["."] [workspace.package] -version = "1.821.0" +version = "1.825.0" edition = "2021" authors = ["Ruben Fiszel "] diff --git a/backend/src/main.rs b/backend/src/main.rs index 23978d7e62..81104d6764 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -44,28 +44,27 @@ use windmill_common::{ CRITICAL_ALERTS_ON_TOKEN_EXPIRY_SETTING, CRITICAL_ALERT_MUTE_UI_SETTING, CRITICAL_ALERT_MUTE_ZOMBIE_JOB_RESTART_SETTING, CRITICAL_ERROR_CHANNELS_SETTING, CUSTOM_TAGS_SETTING, DEFAULT_TAGS_PER_WORKSPACE_SETTING, DEFAULT_TAGS_WORKSPACES_SETTING, - DISABLE_PASSWORD_LOGIN_SETTING, EMAIL_DOMAIN_SETTING, ENV_SETTINGS, - EXPOSE_DEBUG_METRICS_SETTING, EXPOSE_METRICS_SETTING, EXTRA_PIP_INDEX_URL_SETTING, - FORK_WORKSPACE_TAG_APPEND_FORK_SUFFIX_SETTING, + DEPENDENCY_JOB_TAG_SETTING, DISABLE_PASSWORD_LOGIN_SETTING, EMAIL_DOMAIN_SETTING, + ENV_SETTINGS, EXPOSE_DEBUG_METRICS_SETTING, EXPOSE_METRICS_SETTING, + EXTRA_PIP_INDEX_URL_SETTING, FORK_WORKSPACE_TAG_APPEND_FORK_SUFFIX_SETTING, HTTP_ROUTE_DEFAULT_ALLOWED_ORIGINS_SETTING, HTTP_ROUTE_WORKSPACED_ROUTE_SETTING, HUB_API_SECRET_SETTING, HUB_BASE_URL_SETTING, INDEXER_SETTING, INSTANCE_EVENTS_WEBHOOK_SETTING, INSTANCE_PYTHON_VERSION_SETTING, JOB_DEFAULT_TIMEOUT_SECS_SETTING, JOB_ISOLATION_SETTING, JWT_SECRET_SETTING, KEEP_JOB_DIR_SETTING, LICENSE_KEY_SETTING, MAVEN_REPOS_SETTING, MAVEN_SETTINGS_XML_SETTING, MCP_DISABLE_TOKEN_QUERY_PARAM_SETTING, MONITOR_LOGS_ON_OBJECT_STORE_SETTING, - NO_DEFAULT_MAVEN_SETTING, - NPM_CONFIG_REGISTRY_SETTING, NSJAIL_TMPFS_SIZE_MB_SETTING, NSJAIL_TMP_BACKING_SETTING, - NUGET_CONFIG_SETTING, OAUTH_SETTING, OTEL_SETTING, OTEL_TRACES_RETENTION_SECS_SETTING, - OTEL_TRACING_PROXY_SETTING, PIP_INDEX_URL_SETTING, POWERSHELL_REPO_PAT_SETTING, - POWERSHELL_REPO_URL_SETTING, PREVIEW_TAGS_OVERRIDE_SETTING, REQUEST_SIZE_LIMIT_SETTING, - REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING, RESTART_COORDINATION_SETTING, - RETENTION_PERIOD_SECS_OVERRIDES_SETTING, RETENTION_PERIOD_SECS_SETTING, RUBY_REPOS_SETTING, - SAML_METADATA_SETTING, SANDBOX_IMAGE_CACHE_MAX_MB_SETTING, - SANDBOX_IMAGE_DEFAULT_REGISTRY_SETTING, SANDBOX_IMAGE_MAX_SIZE_MB_SETTING, - SANDBOX_IMAGE_PULL_POLICY_SETTING, SANDBOX_REGISTRY_AUTH_SETTING, SCIM_TOKEN_SETTING, - SERVICE_LOG_RETENTION_SECS_SETTING, SMTP_SETTING, STORE_AUDIT_LOGS_S3_SETTING, - TEAMS_SETTING, TIMEOUT_WAIT_RESULT_SETTING, UV_EXCLUDE_NEWER_SETTING, - UV_INDEX_STRATEGY_SETTING, UV_PYTHON_INSTALL_MIRROR_SETTING, + NO_DEFAULT_MAVEN_SETTING, NPM_CONFIG_REGISTRY_SETTING, NSJAIL_TMPFS_SIZE_MB_SETTING, + NSJAIL_TMP_BACKING_SETTING, NUGET_CONFIG_SETTING, OAUTH_SETTING, OTEL_SETTING, + OTEL_TRACES_RETENTION_SECS_SETTING, OTEL_TRACING_PROXY_SETTING, PIP_INDEX_URL_SETTING, + POWERSHELL_REPO_PAT_SETTING, POWERSHELL_REPO_URL_SETTING, PREVIEW_TAGS_OVERRIDE_SETTING, + REQUEST_SIZE_LIMIT_SETTING, REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING, + RESTART_COORDINATION_SETTING, RETENTION_PERIOD_SECS_OVERRIDES_SETTING, + RETENTION_PERIOD_SECS_SETTING, RUBY_REPOS_SETTING, SAML_METADATA_SETTING, + SANDBOX_IMAGE_CACHE_MAX_MB_SETTING, SANDBOX_IMAGE_DEFAULT_REGISTRY_SETTING, + SANDBOX_IMAGE_MAX_SIZE_MB_SETTING, SANDBOX_IMAGE_PULL_POLICY_SETTING, + SANDBOX_REGISTRY_AUTH_SETTING, SCIM_TOKEN_SETTING, SERVICE_LOG_RETENTION_SECS_SETTING, + SMTP_SETTING, STORE_AUDIT_LOGS_S3_SETTING, TEAMS_SETTING, TIMEOUT_WAIT_RESULT_SETTING, + UV_EXCLUDE_NEWER_SETTING, UV_INDEX_STRATEGY_SETTING, UV_PYTHON_INSTALL_MIRROR_SETTING, WORKSPACE_FAIRNESS_DURATION_SECS_SETTING, WORKSPACE_FAIRNESS_ENABLED_SETTING, WORKSPACE_FAIRNESS_MAX_PERCENT_SETTING, WORKSPACE_FAIRNESS_MIN_TOTAL_SETTING, WORKSPACE_MAX_QUEUED_JOBS_SETTING, WORKSPACE_REGISTRIES_SETTING, @@ -126,24 +125,23 @@ use windmill_worker::{ }; use crate::monitor::{ - initial_load, load_concurrency_key_max_queued, load_disable_password_login, - load_fork_workspace_tag_append_fork_suffix, load_keep_job_dir, - load_mcp_disable_token_query_param, load_metrics_debug_enabled, - load_preview_tags_override, load_require_preexisting_user, load_retention_period_overrides, - load_tag_per_workspace_enabled, load_tag_per_workspace_workspaces, - load_workspace_fairness_duration_secs, load_workspace_fairness_enabled, - load_workspace_fairness_max_percent, load_workspace_fairness_min_total, - load_workspace_max_queued_jobs, monitor_db, reload_app_workspaced_route_setting, - reload_audit_log_retention_days_setting, reload_base_url_setting, - reload_bun_install_min_release_age_setting, reload_bunfig_install_scopes_setting, - reload_critical_alert_mute_ui_setting, reload_critical_alert_mute_zombie_job_restart_setting, + initial_load, load_concurrency_key_max_queued, load_dependency_job_tag, + load_disable_password_login, load_fork_workspace_tag_append_fork_suffix, load_keep_job_dir, + load_mcp_disable_token_query_param, load_metrics_debug_enabled, load_preview_tags_override, + load_require_preexisting_user, load_retention_period_overrides, load_tag_per_workspace_enabled, + load_tag_per_workspace_workspaces, load_workspace_fairness_duration_secs, + load_workspace_fairness_enabled, load_workspace_fairness_max_percent, + load_workspace_fairness_min_total, load_workspace_max_queued_jobs, monitor_db, + reload_app_workspaced_route_setting, reload_audit_log_retention_days_setting, + reload_base_url_setting, reload_bun_install_min_release_age_setting, + reload_bunfig_install_scopes_setting, reload_critical_alert_mute_ui_setting, + reload_critical_alert_mute_zombie_job_restart_setting, reload_critical_alerts_on_token_expiry_setting, reload_critical_error_channels_setting, reload_extra_pip_index_url_setting, reload_http_route_default_allowed_origins_setting, - reload_http_route_workspaced_route_setting, - reload_hub_api_secret_setting, reload_hub_base_url_setting, - reload_instance_events_webhook_setting, reload_job_default_timeout_setting, - reload_job_isolation_setting, reload_jwt_secret_setting, reload_license_key, - reload_npm_config_registry_setting, reload_nsjail_tmp_backing_setting, + reload_http_route_workspaced_route_setting, reload_hub_api_secret_setting, + reload_hub_base_url_setting, reload_instance_events_webhook_setting, + reload_job_default_timeout_setting, reload_job_isolation_setting, reload_jwt_secret_setting, + reload_license_key, reload_npm_config_registry_setting, reload_nsjail_tmp_backing_setting, reload_nsjail_tmpfs_size_setting, reload_otel_traces_retention_secs_setting, reload_otel_tracing_proxy_setting, reload_pip_index_url_setting, reload_retention_period_setting, reload_sandbox_image_cache_max_setting, @@ -423,7 +421,6 @@ struct HubResourceTypeRaw { pub display_name: Option>, } - /// Processed resource type with parsed schema #[derive(serde::Deserialize, serde::Serialize, Clone)] pub struct HubResourceType { @@ -1889,12 +1886,10 @@ async fn process_notify_event( ); windmill_common::workspaces::PUBLIC_APP_RATE_LIMIT_CACHE.remove(payload); } - windmill_common::SCRIPT_VERSION_DELETED_CHANNEL => { - match serde_json::from_str(payload) { - Ok(deleted) => windmill_api_scripts::scripts::evict_deleted_script_versions(deleted), - Err(e) => tracing::error!("Invalid script version deletion payload {payload}: {e}"), - } - } + windmill_common::SCRIPT_VERSION_DELETED_CHANNEL => match serde_json::from_str(payload) { + Ok(deleted) => windmill_api_scripts::scripts::evict_deleted_script_versions(deleted), + Err(e) => tracing::error!("Invalid script version deletion payload {payload}: {e}"), + }, "notify_runnable_version_change" => { tracing::info!("Runnable version change detected: {}", payload); match payload.split(':').collect::>().as_slice() { @@ -2055,6 +2050,11 @@ async fn process_notify_event( tracing::error!("Error loading preview tags override: {e:#}"); } } + DEPENDENCY_JOB_TAG_SETTING => { + if let Err(e) = load_dependency_job_tag(db).await { + tracing::error!("Error loading dependency job tag: {e:#}"); + } + } WORKSPACE_FAIRNESS_ENABLED_SETTING => { if let Err(e) = load_workspace_fairness_enabled(db).await { tracing::error!("Error loading workspace fairness enabled: {e:#}"); diff --git a/backend/src/monitor.rs b/backend/src/monitor.rs index 751d19fecc..77963b73c6 100644 --- a/backend/src/monitor.rs +++ b/backend/src/monitor.rs @@ -60,17 +60,17 @@ use windmill_common::{ CRITICAL_ALERTS_ON_DB_OVERSIZE_SETTING, CRITICAL_ALERTS_ON_TOKEN_EXPIRY_SETTING, CRITICAL_ALERT_MUTE_UI_SETTING, CRITICAL_ALERT_MUTE_ZOMBIE_JOB_RESTART_SETTING, CRITICAL_ERROR_CHANNELS_SETTING, CUSTOM_TAGS_SETTING, DEFAULT_TAGS_PER_WORKSPACE_SETTING, - DEFAULT_TAGS_WORKSPACES_SETTING, DISABLE_PASSWORD_LOGIN, DISABLE_PASSWORD_LOGIN_SETTING, - EXPOSE_DEBUG_METRICS_SETTING, EXPOSE_METRICS_SETTING, EXTRA_PIP_INDEX_URL_SETTING, - FORK_WORKSPACE_TAG_APPEND_FORK_SUFFIX_SETTING, HUB_API_SECRET_SETTING, - HUB_BASE_URL_SETTING, INSTANCE_PYTHON_VERSION_SETTING, JOB_DEFAULT_TIMEOUT_SECS_SETTING, - JOB_ISOLATION_SETTING, JWT_SECRET_SETTING, KEEP_JOB_DIR_SETTING, LICENSE_KEY_SETTING, - MCP_DISABLE_TOKEN_QUERY_PARAM, MCP_DISABLE_TOKEN_QUERY_PARAM_SETTING, - MONITOR_LOGS_ON_OBJECT_STORE_SETTING, NPMRC_SETTING, NPM_CONFIG_REGISTRY_SETTING, - NSJAIL_TMPFS_SIZE_MB_SETTING, NSJAIL_TMP_BACKING_SETTING, NUGET_CONFIG_SETTING, - OTEL_SETTING, OTEL_TRACES_RETENTION_SECS_SETTING, OTEL_TRACING_PROXY_SETTING, - PIP_INDEX_URL_SETTING, POWERSHELL_REPO_PAT_SETTING, POWERSHELL_REPO_URL_SETTING, - PREVIEW_TAGS_OVERRIDE_SETTING, REQUEST_SIZE_LIMIT_SETTING, + DEFAULT_TAGS_WORKSPACES_SETTING, DEPENDENCY_JOB_TAG_SETTING, DISABLE_PASSWORD_LOGIN, + DISABLE_PASSWORD_LOGIN_SETTING, EXPOSE_DEBUG_METRICS_SETTING, EXPOSE_METRICS_SETTING, + EXTRA_PIP_INDEX_URL_SETTING, FORK_WORKSPACE_TAG_APPEND_FORK_SUFFIX_SETTING, + HUB_API_SECRET_SETTING, HUB_BASE_URL_SETTING, INSTANCE_PYTHON_VERSION_SETTING, + JOB_DEFAULT_TIMEOUT_SECS_SETTING, JOB_ISOLATION_SETTING, JWT_SECRET_SETTING, + KEEP_JOB_DIR_SETTING, LICENSE_KEY_SETTING, MCP_DISABLE_TOKEN_QUERY_PARAM, + MCP_DISABLE_TOKEN_QUERY_PARAM_SETTING, MONITOR_LOGS_ON_OBJECT_STORE_SETTING, NPMRC_SETTING, + NPM_CONFIG_REGISTRY_SETTING, NSJAIL_TMPFS_SIZE_MB_SETTING, NSJAIL_TMP_BACKING_SETTING, + NUGET_CONFIG_SETTING, OTEL_SETTING, OTEL_TRACES_RETENTION_SECS_SETTING, + OTEL_TRACING_PROXY_SETTING, PIP_INDEX_URL_SETTING, POWERSHELL_REPO_PAT_SETTING, + POWERSHELL_REPO_URL_SETTING, PREVIEW_TAGS_OVERRIDE_SETTING, REQUEST_SIZE_LIMIT_SETTING, REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING, RETENTION_PERIOD_SECS_SETTING, SAML_METADATA_SETTING, SANDBOX_IMAGE_CACHE_MAX_MB_SETTING, SANDBOX_IMAGE_DEFAULT_REGISTRY_SETTING, SANDBOX_IMAGE_MAX_SIZE_MB_SETTING, @@ -93,11 +93,11 @@ use windmill_common::{ load_periodic_bash_script_interval_from_env, load_whitelist_env_vars_from_env, load_worker_config, store_pull_query, store_suspended_pull_query, Connection, WorkerConfig, CLOUD_HOSTED, CONCURRENCY_KEY_MAX_QUEUED, CONCURRENCY_KEY_MAX_QUEUED_DEFAULT, - DEFAULT_TAGS_PER_WORKSPACE, DEFAULT_TAGS_WORKSPACES, FORK_WORKSPACE_TAG_APPEND_FORK_SUFFIX, - INDEXER_CONFIG, PREVIEW_TAGS_OVERRIDE, SMTP_CONFIG, WINDMILL_DIR, WORKER_CONFIG, - WORKER_GROUP, WORKSPACE_FAIRNESS_DURATION_SECS, WORKSPACE_FAIRNESS_ENABLED, - WORKSPACE_FAIRNESS_MAX_PERCENT, WORKSPACE_FAIRNESS_MIN_TOTAL, WORKSPACE_MAX_QUEUED_JOBS, - WORKSPACE_MAX_QUEUED_JOBS_DEFAULT, + DEFAULT_TAGS_PER_WORKSPACE, DEFAULT_TAGS_WORKSPACES, DEPENDENCY_JOB_TAG, + FORK_WORKSPACE_TAG_APPEND_FORK_SUFFIX, INDEXER_CONFIG, PREVIEW_TAGS_OVERRIDE, SMTP_CONFIG, + WINDMILL_DIR, WORKER_CONFIG, WORKER_GROUP, WORKSPACE_FAIRNESS_DURATION_SECS, + WORKSPACE_FAIRNESS_ENABLED, WORKSPACE_FAIRNESS_MAX_PERCENT, WORKSPACE_FAIRNESS_MIN_TOTAL, + WORKSPACE_MAX_QUEUED_JOBS, WORKSPACE_MAX_QUEUED_JOBS_DEFAULT, }, KillpillSender, AUDIT_LOG_RETENTION_DAYS, BASE_URL, CRITICAL_ALERTS_ON_DB_OVERSIZE, CRITICAL_ALERTS_ON_TOKEN_EXPIRY, CRITICAL_ALERT_MUTE_UI_ENABLED, @@ -312,6 +312,9 @@ pub async fn initial_load( pass.setting(PREVIEW_TAGS_OVERRIDE_SETTING, false, |v| async move { apply_preview_tags_override(v) }); + pass.setting(DEPENDENCY_JOB_TAG_SETTING, false, |v| async move { + apply_dependency_job_tag(v) + }); // Load per-workspace retention overrides before the first cleanup tick so a fresh server // never sweeps globally without honoring configured longer-retention workspaces. @@ -876,6 +879,24 @@ pub fn apply_preview_tags_override(value: Option) { } } +pub async fn load_dependency_job_tag(db: &DB) -> error::Result<()> { + let v = load_value_from_global_settings(db, DEPENDENCY_JOB_TAG_SETTING).await?; + apply_dependency_job_tag(v); + Ok(()) +} + +pub fn apply_dependency_job_tag(value: Option) { + let tag = match value { + Some(serde_json::Value::String(t)) if !t.trim().is_empty() => Some(t.trim().to_string()), + None | Some(serde_json::Value::Null) | Some(serde_json::Value::String(_)) => None, + Some(other) => { + tracing::error!("{DEPENDENCY_JOB_TAG_SETTING} is not a string: {other}, ignoring"); + return; + } + }; + DEPENDENCY_JOB_TAG.store(std::sync::Arc::new(tag)); +} + // Upper bound on the duration window. Postgres `make_interval(secs => $1::int4)` is the consumer // downstream, so this stays comfortably below `i32::MAX` and the subsequent `u32 -> i32` cast in // `workspace_fairness::refresh_overloaded` cannot wrap into a negative interval (which would @@ -6146,6 +6167,21 @@ async fn handle_zombie_jobs(db: &Pool, base_internal_url: &str, node_n continue; } if let Some(job) = job.unwrap() { + // Read while the job is still queued: a re-run (perpetual, retry) takes its cap from + // here once the `job_perms` row is swept after completion. A failed read leaves the + // job for the next sweep rather than completing it with no cap. + let perms = + match windmill_common::auth::get_job_perms(db, &job.id, &job.workspace_id).await { + Ok(perms) => perms, + Err(e) => { + tracing::error!( + "Could not read the permissions of zombie job {}: {e:#}", + job.id + ); + continue; + } + }; + let job_token_scopes = perms.as_ref().and_then(|p| p.job_token_scopes.clone()); let label = ephemeral_script_token_label(&job.permissioned_as, &job.created_by); let token = create_token_for_owner( &db, @@ -6155,7 +6191,7 @@ async fn handle_zombie_jobs(db: &Pool, base_internal_url: &str, node_n job_token_expiry_secs(&db, &job.workspace_id).await, &job.permissioned_as_email, &job.id, - None, + perms, Some(format!("handle_zombie_jobs")), ) .await @@ -6177,10 +6213,12 @@ async fn handle_zombie_jobs(db: &Pool, base_internal_url: &str, node_n ); let memory_peak = job.memory_peak.unwrap_or(0); let (_, killpill_rx_never_used) = KillpillSender::new(1); + let mut completed = windmill_queue::MiniCompletedJob::from(job); + completed.job_token_scopes = job_token_scopes; let _ = handle_job_error( db, &client, - &windmill_queue::MiniCompletedJob::from(job), + &completed, memory_peak, None, error::Error::ExecutionErr(error_message.clone()), diff --git a/backend/summarized_schema.txt b/backend/summarized_schema.txt index 169eb81681..2a518a7cc8 100644 --- a/backend/summarized_schema.txt +++ b/backend/summarized_schema.txt @@ -97,7 +97,7 @@ draft: workspace_id(char), path(char), typ(draft_type), value(json), created_at( email_to_igroup: email(char), igroup(char) email_trigger: path(char), local_part(char), workspaced_local_part(bool), script_path(char), is_flow(bool), workspace_id(char), edited_by(char), email(char), edited_at(ts), extra_perms(jsonb), error_handler_path(char), error_handler_args(jsonb), retry(jsonb), mode(trigger_mode), labels(text[]) favorite: usr(char), workspace_id(char), path(char), favorite_kind(favorite_kind) -flow: workspace_id(char), path(char), summary(text), description(text), value(jsonb), edited_by(char), edited_at(ts), archived(bool), schema(json), extra_perms(jsonb), dependency_job(uuid), draft_only(bool), tag(char), ws_error_handler_muted(bool), dedicated_worker(bool), timeout(int), visible_to_runner_only(bool), concurrency_key(char), versions(bigint[]), on_behalf_of(varchar), on_behalf_of_email(text), lock_error_logs(text), labels(text[]) +flow: workspace_id(char), path(char), summary(text), description(text), value(jsonb), edited_by(char), edited_at(ts), archived(bool), schema(json), extra_perms(jsonb), dependency_job(uuid), draft_only(bool), tag(char), ws_error_handler_muted(bool), dedicated_worker(bool), timeout(int), visible_to_runner_only(bool), concurrency_key(char), versions(bigint[]), on_behalf_of(varchar), on_behalf_of_email(text), lock_error_logs(text), labels(text[]), job_token_scopes(text[]) FK: (workspace_id) -> workspace(id) flow_conversation: id(uuid), workspace_id(char), flow_path(char), title(char), created_at(ts), updated_at(ts), created_by(char), is_test(bool) FK: (workspace_id) -> workspace(id) @@ -132,7 +132,7 @@ input: id(uuid), workspace_id(char), runnable_id(char), runnable_type(runnable_t FK: (workspace_id) -> workspace(id) instance_group: name(char), summary(char), id(char), scim_display_name(char), external_id(char) job_logs: job_id(uuid), workspace_id(char), created_at(ts), logs(text), log_offset(int), log_file_index(text[]) -job_perms: job_id(uuid), email(char), username(char), is_admin(bool), is_operator(bool), created_at(ts), workspace_id(char), groups(text[]), folders(jsonb[]), end_user_email(char) +job_perms: job_id(uuid), email(char), username(char), is_admin(bool), is_operator(bool), created_at(ts), workspace_id(char), groups(text[]), folders(jsonb[]), end_user_email(char), job_token_scopes(text[]) job_resolution: job_id(uuid), workspace_id(char), resolved_at(ts), resolved_by(char), note(text), automatic(bool) job_result_stream: job_id(uuid), workspace_id(text), stream(text) job_result_stream_v2: job_id(uuid), workspace_id(text), stream(text), idx(int) @@ -183,7 +183,7 @@ resume_job: id(uuid), job(uuid), flow(uuid), created_at(ts), value(jsonb), appro runnable_settings: hash(bigint), debouncing_settings(bigint), concurrency_settings(bigint) schedule: workspace_id(char), path(char), edited_by(char), edited_at(ts), schedule(char), enabled(bool), script_path(char), args(jsonb), extra_perms(jsonb), is_flow(bool), email(char), error(text), timezone(char), on_failure(char), on_recovery(char), on_failure_times(int), on_failure_exact(bool), on_failure_extra_args(jsonb), on_recovery_times(int), on_recovery_extra_args(jsonb), ws_error_handler_muted(bool), retry(jsonb), summary(char), no_flow_overlap(bool), tag(char), paused_until(ts), on_success(char), on_success_extra_args(jsonb), cron_version(text), description(text), dynamic_skip(char), labels(text[]) FK: (workspace_id) -> workspace(id) -script: workspace_id(char), hash(bigint), path(char), parent_hashes(bigint[]), summary(text), description(text), content(text), created_by(char), created_at(ts), archived(bool), schema(json), deleted(bool), is_template(bool), extra_perms(jsonb), lock(text), lock_error_logs(text), language(script_lang), kind(script_kind), tag(char), draft_only(bool), envs(char), concurrent_limit(int), concurrency_time_window_s(int), cache_ttl(int), dedicated_worker(bool), ws_error_handler_muted(bool), priority(smallint), timeout(int), delete_after_use(bool), restart_unless_cancelled(bool), concurrency_key(char), visible_to_runner_only(bool), auto_kind(varchar), codebase(char), has_preprocessor(bool), schema_validation(bool), assets(jsonb), debounce_key(char), debounce_delay_s(int), cache_ignore_s3_path(bool), runnable_settings_handle(bigint), labels(text[]), on_behalf_of(varchar), on_behalf_of_email(text) +script: workspace_id(char), hash(bigint), path(char), parent_hashes(bigint[]), summary(text), description(text), content(text), created_by(char), created_at(ts), archived(bool), schema(json), deleted(bool), is_template(bool), extra_perms(jsonb), lock(text), lock_error_logs(text), language(script_lang), kind(script_kind), tag(char), draft_only(bool), envs(char), concurrent_limit(int), concurrency_time_window_s(int), cache_ttl(int), dedicated_worker(bool), ws_error_handler_muted(bool), priority(smallint), timeout(int), delete_after_use(bool), restart_unless_cancelled(bool), concurrency_key(char), visible_to_runner_only(bool), auto_kind(varchar), codebase(char), has_preprocessor(bool), schema_validation(bool), assets(jsonb), debounce_key(char), debounce_delay_s(int), cache_ignore_s3_path(bool), runnable_settings_handle(bigint), labels(text[]), on_behalf_of(varchar), on_behalf_of_email(text), job_token_scopes(text[]) FK: (workspace_id) -> workspace(id) skip_workspace_diff_tally: workspace_id(char), added_at(ts) sqs_trigger: path(char), queue_url(char), aws_resource_path(char), message_attributes(text[]), script_path(char), is_flow(bool), workspace_id(char), edited_by(char), email(char), edited_at(ts), extra_perms(jsonb), error(text), server_id(char), last_server_ping(ts), aws_auth_resource_type(aws_auth_resource_type), error_handler_path(char), error_handler_args(jsonb), retry(jsonb), mode(trigger_mode), labels(text[]) diff --git a/backend/tests/ai_agent_nul.rs b/backend/tests/ai_agent_nul.rs new file mode 100644 index 0000000000..1af46ecc82 --- /dev/null +++ b/backend/tests/ai_agent_nul.rs @@ -0,0 +1,125 @@ +use axum::{routing::post, Json, Router}; +use serde_json::{json, Value}; +use sqlx::{Pool, Postgres}; +use std::time::Duration; +use windmill_common::{flows::FlowValue, jobs::JobPayload}; +use windmill_test_utils::{ + completed_job, in_test_worker, listen_for_completed_jobs, ApiServer, RunJob, StreamFind, +}; + +const MEMORY_ID: &str = "0199c0de-0000-7000-8000-000000000001"; + +/// A model that puts a U+0000 in its tool call arguments, then in its answer. +async fn model(Json(body): Json) -> ([(&'static str, &'static str); 1], String) { + let finished = body["messages"] + .as_array() + .unwrap() + .iter() + .any(|m| m["role"] == "tool"); + let delta = if finished { + json!({"role":"assistant", "content":"are\u{0}tired"}) + } else { + json!({"role":"assistant", "tool_calls":[{ + "index":0, "id":"call_0", "type":"function", + "function":{"name":"echo", "arguments":"{\"x\":\"a\\u0000b\"}"} + }]}) + }; + let event = json!({"choices":[{"index":0, "delta":delta, "finish_reason":null}]}); + let end = json!({"choices":[{"index":0, "delta":{}, "finish_reason":if finished {"stop"} else {"tool_calls"}}]}); + ( + [("content-type", "text/event-stream")], + format!("data: {event}\n\ndata: {end}\n\ndata: [DONE]\n\n"), + ) +} + +async fn run_flow(db: &Pool, modules: Value) -> anyhow::Result { + let server = ApiServer::start(db.clone()).await?; + let flow: FlowValue = serde_json::from_value(json!({ "modules": modules }))?; + let id = RunJob::from(JobPayload::RawFlow { + value: flow, + path: Some("u/test/nul".into()), + restarted_from: None, + }) + .push(db) + .await; + let notifications = listen_for_completed_jobs(db).await; + tokio::time::timeout( + Duration::from_secs(45), + in_test_worker(db, notifications.find(&id), server.addr.port()), + ) + .await?; + let job = completed_job(id, db).await; + server.close().await?; + assert!(job.success, "{:?}", job.result); + Ok(job.json_result().expect("flow result")) +} + +#[sqlx::test(fixtures("base"))] +#[serial_test::serial] +async fn agent_step_survives_nul_in_model_output(db: Pool) -> anyhow::Result<()> { + std::env::set_var("ALLOW_PRIVATE_AI_BASE_URLS", "true"); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await?; + let base = format!("http://{}", listener.local_addr()?); + let stub = tokio::spawn(async move { + let router = Router::new().route("/v1/chat/completions", post(model)); + axum::serve(listener, router).await.unwrap(); + }); + let result = run_flow( + &db, + json!([{"id":"agent","value":{ + "type":"aiagent", + "tools":[{"id":"t0","summary":"echo","value":{ + "type":"rawscript", "language":"bun", "input_transforms":{}, + "content":"export async function main(x: string) { return x }" + }}], + "input_transforms":{ + "provider":{"type":"static","value":{"kind":"customai","model":"nul-test","resource":{"base_url":format!("{base}/v1")}}}, + "user_message":{"type":"static","value":"echo"}, + "memory":{"type":"static","value":{"kind":"auto","context_length":10,"memory_id":MEMORY_ID}} + } + }}]), + ) + .await?; + stub.abort(); + assert_eq!(result["output"], "aretired"); + let tool_message = result["messages"] + .as_array() + .unwrap() + .iter() + .find(|m| m["role"] == "tool") + .expect("tool message"); + assert_eq!(tool_message["content"], "\"ab\""); + // The model's own words are also written to a jsonb column the job result never passes through. + let memory: String = sqlx::query_scalar( + "SELECT messages::text FROM ai_agent_memory WHERE conversation_id = $1::uuid", + ) + .bind(MEMORY_ID) + .fetch_one(&db) + .await?; + assert!(memory.contains("aretired"), "{memory}"); + Ok(()) +} + +/// A step's args are evaluated from the previous step's result as the worker holds it, not as +/// the completed job stored it. +#[sqlx::test(fixtures("base"))] +#[serial_test::serial] +async fn flow_step_args_survive_nul_in_previous_result(db: Pool) -> anyhow::Result<()> { + let result = run_flow( + &db, + json!([ + {"id":"a","value":{ + "type":"rawscript", "language":"bun", "input_transforms":{}, + "content":"export async function main() { return 'a\\u0000b' }" + }}, + {"id":"b","value":{ + "type":"rawscript", "language":"bun", + "input_transforms":{"x":{"type":"javascript","expr":"results.a"}}, + "content":"export async function main(x: string) { return x }" + }} + ]), + ) + .await?; + assert_eq!(result, "ab"); + Ok(()) +} diff --git a/backend/tests/ai_decision.rs b/backend/tests/ai_decision.rs new file mode 100644 index 0000000000..98c219dfb2 --- /dev/null +++ b/backend/tests/ai_decision.rs @@ -0,0 +1,129 @@ +//! AI decision steps run end to end against a stand-in for System One endpoints: TypeSafe's, and +//! Cloudflare Workers AI's, which serves each model at its own URL inside an API envelope. + +use axum::{routing::post, Json, Router}; +use serde_json::{json, Value}; +use sqlx::{Pool, Postgres}; +use windmill_common::{flows::FlowValue, jobs::JobPayload}; +use windmill_test_utils::*; + +/// Answers every question as a choice of `refund`, the way TypeSafe shapes an answer. +fn refund_answers(body: &Value) -> Value { + let answers = body["questions"] + .as_object() + .into_iter() + .flatten() + .map(|(name, _)| { + let answer = json!({ + "type": "choice", + "choice": "refund", + "probabilities": {"refund": 0.9, "bug": 0.1}, + "confidence": 0.8 + }); + (name.clone(), answer) + }) + .collect::>(); + json!({"model": body["model"], "answers": answers, "usage": {"input_tokens": 10}}) +} + +async fn start_decision_stub() -> anyhow::Result { + let app = Router::new() + .route( + "/v1/systemone", + post(|Json(body): Json| async move { Json(refund_answers(&body)) }), + ) + .route( + "/accounts/acc/ai/run/@cf/cloudflare/clef-flash", + post(|Json(body): Json| async move { + Json(json!({"result": refund_answers(&body), "success": true, "errors": []})) + }), + ); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await?; + let port = listener.local_addr()?.port(); + tokio::spawn(async move { axum::serve(listener, app).await }); + Ok(port) +} + +/// A decision's answers reach later steps as `results..output`, which a Branch to one after it +/// routes on. +#[cfg(feature = "quickjs")] +#[sqlx::test(fixtures("base"))] +async fn test_ai_decision_answers_route_a_following_branch( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + // The stub listens on loopback, which the SSRF check refuses without this. + std::env::set_var("ALLOW_PRIVATE_AI_BASE_URLS", "true"); + let stub_port = start_decision_stub().await?; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + for provider in [ + json!({ + "kind": "typesafe", + "resource": {"api_key": "key", "base_url": format!("http://127.0.0.1:{stub_port}/v1")}, + "model": "jev-latest" + }), + json!({ + "kind": "cloudflare", + "resource": { + "token": "key", + "base_url": format!("http://127.0.0.1:{stub_port}/accounts/acc/ai/run/@cf/cloudflare") + }, + "model": "clef-flash" + }), + ] { + let flow: FlowValue = serde_json::from_value(json!({ + "modules": [ + {"id": "d", "value": { + "type": "aidecision", + "input_transforms": { + "provider": {"type": "static", "value": provider}, + "state": {"type": "static", "value": "I was charged twice"}, + "questions": {"type": "static", "value": {"intent": { + "type": "choice", + "instructions": "What does the customer want?", + "criteria": {"refund": "Money back", "bug": "Something is broken"} + }}} + } + }}, + {"id": "route", "value": { + "type": "branchone", + "branches": [ + { + "expr": "results.d.output.intent.choice === 'bug'", + "modules": [{"id": "b", "value": {"type": "identity"}}] + }, + { + "expr": "results.d.output.intent.choice === 'refund'", + "modules": [{"id": "r", "value": { + "type": "rawscript", + "language": "deno", + "content": "export function main(choice){ return 'handled ' + choice }", + "input_transforms": {"choice": { + "type": "javascript", + "expr": "results.d.output.intent.choice" + }} + }}] + } + ], + "default": [] + }} + ] + }))?; + + let job = run_job_in_new_worker_until_complete( + &db, + false, + JobPayload::RawFlow { value: flow, path: None, restarted_from: None }, + port, + ) + .await; + assert_eq!( + job.json_result(), + Some(json!("handled refund")), + "{provider}" + ); + } + Ok(()) +} diff --git a/backend/tests/asset_trigger_dispatch.rs b/backend/tests/asset_trigger_dispatch.rs index c165958a33..08d755f5d1 100644 --- a/backend/tests/asset_trigger_dispatch.rs +++ b/backend/tests/asset_trigger_dispatch.rs @@ -245,6 +245,7 @@ fn make_mini(id: Uuid, runnable_path: &str) -> MiniCompletedJob { cache_ignore_s3_path: None, runnable_settings_handle: None, build_binary_only: false, + job_token_scopes: None, } } @@ -330,6 +331,7 @@ async fn end_to_end_asset_dispatch(db: Pool) -> anyhow::Result<()> { concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default(), debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), labels: None, + job_token_scopes: None, }; let completed = RunJob::from(job).run_until_complete(&db, false, port).await; assert!( @@ -493,6 +495,7 @@ async fn partition_dynamic_resolved_persisted_and_propagated( concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default(), debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), labels: None, + job_token_scopes: None, }; let completed = RunJob::from(job) .arg("tenant_id", json!("acme")) diff --git a/backend/tests/batch_rerun.rs b/backend/tests/batch_rerun.rs index 51a6afa565..58cf27db87 100644 --- a/backend/tests/batch_rerun.rs +++ b/backend/tests/batch_rerun.rs @@ -53,6 +53,7 @@ fn script_payload() -> JobPayload { concurrency_settings: ConcurrencySettings::default().into(), debouncing_settings: DebouncingSettings::default(), labels: None, + job_token_scopes: None, } } @@ -63,6 +64,7 @@ fn flow_payload() -> JobPayload { apply_preprocessor: false, version: FLOW_VERSION, labels: None, + job_token_scopes: None, } } diff --git a/backend/tests/bun_jobs.rs b/backend/tests/bun_jobs.rs index 14f8c422a4..50fad28a04 100644 --- a/backend/tests/bun_jobs.rs +++ b/backend/tests/bun_jobs.rs @@ -854,6 +854,7 @@ fn run_main_script_job(hash: i64) -> RunJob { concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default(), debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), labels: None, + job_token_scopes: None, }) } diff --git a/backend/tests/caller_modules_strip.rs b/backend/tests/caller_modules_strip.rs index e6f96c04ec..a65ca22d1b 100644 --- a/backend/tests/caller_modules_strip.rs +++ b/backend/tests/caller_modules_strip.rs @@ -54,6 +54,7 @@ async fn stored_modules(db: &Pool, payload: JobPayload) -> Option) { concurrency_settings: ConcurrencySettings::default(), debouncing_settings: DebouncingSettings::default(), labels: None, + job_token_scopes: None, }; assert_eq!(stored_modules(&db, deployed).await, None); diff --git a/backend/tests/dedicated_workers.rs b/backend/tests/dedicated_workers.rs index 13fb043454..5e9c371f90 100644 --- a/backend/tests/dedicated_workers.rs +++ b/backend/tests/dedicated_workers.rs @@ -85,6 +85,7 @@ mod dedicated_worker_tests { apply_preprocessor: false, labels: None, version: 3000000000000001, + job_token_scopes: None, }) .arg("x", json!(5)) .push(&db) @@ -122,6 +123,7 @@ mod dedicated_worker_tests { apply_preprocessor: false, labels: None, version: 3000000000000002, + job_token_scopes: None, }) .arg("x", json!(7)) .push(&db) @@ -161,6 +163,7 @@ mod dedicated_worker_tests { apply_preprocessor: false, labels: None, version: 3000000000000003, + job_token_scopes: None, }) .arg("x", json!(5)) .push(&db) @@ -205,6 +208,7 @@ mod dedicated_worker_tests { concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default( ), debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), + job_token_scopes: None, }) .arg("x", json!(21)) .push(&db) @@ -247,6 +251,7 @@ mod dedicated_worker_tests { concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default( ), debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), + job_token_scopes: None, }) .arg("x", json!(5)) .push(&db) @@ -265,6 +270,7 @@ mod dedicated_worker_tests { concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default( ), debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), + job_token_scopes: None, }) .arg("x", json!(5)) .push(&db) @@ -327,6 +333,7 @@ mod dedicated_worker_tests { apply_preprocessor: false, labels: None, version: 3000000000000004, + job_token_scopes: None, }) .run_until_complete(&db, false, port) .await @@ -356,6 +363,7 @@ mod dedicated_worker_tests { apply_preprocessor: false, labels: None, version: 3000000000000005, + job_token_scopes: None, }) .arg("x", json!(5)) .push(&db) @@ -391,6 +399,7 @@ mod dedicated_worker_tests { apply_preprocessor: false, labels: None, version: 3000000000000006, + job_token_scopes: None, }) .arg("x", json!(5)) .push(&db) @@ -427,6 +436,7 @@ mod dedicated_worker_tests { apply_preprocessor: false, labels: None, version: 3000000000000007, + job_token_scopes: None, }) .arg("x", json!(5)) .push(&db) @@ -465,6 +475,7 @@ mod dedicated_worker_tests { apply_preprocessor: false, labels: None, version: 3000000000000008, + job_token_scopes: None, }) .arg("x", json!(5)) .push(&db) @@ -500,6 +511,7 @@ mod dedicated_worker_tests { apply_preprocessor: false, labels: None, version: 3000000000000009, + job_token_scopes: None, }) .arg("x", json!(5)) .push(&db) @@ -511,6 +523,7 @@ mod dedicated_worker_tests { apply_preprocessor: false, labels: None, version: 3000000000000010, + job_token_scopes: None, }) .arg("x", json!(5)) .push(&db) @@ -564,6 +577,7 @@ mod dedicated_worker_tests { concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default( ), debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), + job_token_scopes: None, }) .arg("x", json!(5)) .push(&db) @@ -622,6 +636,7 @@ mod dedicated_worker_tests { concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default( ), debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), + job_token_scopes: None, }) .arg("x", json!(7)) .push(&db) @@ -634,6 +649,7 @@ mod dedicated_worker_tests { apply_preprocessor: false, labels: None, version: 3000000000000011, + job_token_scopes: None, }) .arg("x", json!(7)) .push(&db) @@ -683,6 +699,7 @@ mod dedicated_worker_tests { apply_preprocessor: false, labels: None, version: 3000000000000012, + job_token_scopes: None, }) .arg("x", json!(5)) .push(&db) @@ -742,6 +759,7 @@ mod dedicated_worker_tests { apply_preprocessor: false, labels: None, version: 3000000000000013, + job_token_scopes: None, }) .run_until_complete(&db, false, port) .await @@ -777,6 +795,7 @@ mod dedicated_worker_tests { concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default( ), debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), + job_token_scopes: None, }) .arg("x", json!(5)) .push(&db) @@ -795,6 +814,7 @@ mod dedicated_worker_tests { concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default( ), debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), + job_token_scopes: None, }) .arg("x", json!(5)) .push(&db) @@ -848,6 +868,7 @@ mod dedicated_worker_tests { concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default( ), debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), + job_token_scopes: None, }) .arg("x", json!(5)) .push(&db) @@ -867,6 +888,7 @@ mod dedicated_worker_tests { concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default( ), debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), + job_token_scopes: None, }) .arg("x", json!(5)) .push(&db) @@ -926,6 +948,7 @@ mod dedicated_worker_tests { apply_preprocessor: false, labels: None, version: 3000000000000014, + job_token_scopes: None, }) .arg("x", json!(20)) .push(&db) @@ -938,6 +961,7 @@ mod dedicated_worker_tests { apply_preprocessor: false, labels: None, version: 3000000000000014, + job_token_scopes: None, }) .arg("x", json!(5)) .push(&db) @@ -988,6 +1012,7 @@ mod dedicated_worker_tests { concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default( ), debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), + job_token_scopes: None, }) .arg("x", json!(5)) .push(&db) @@ -1038,6 +1063,7 @@ mod dedicated_worker_tests { concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default( ), debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), + job_token_scopes: None, }) .arg("x", json!(5)) .push(&db) @@ -1089,6 +1115,7 @@ mod dedicated_worker_tests { concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default( ), debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), + job_token_scopes: None, }) .arg("x", json!(5)) .push(&db) @@ -1142,6 +1169,7 @@ mod dedicated_worker_tests { concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default( ), debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), + job_token_scopes: None, }) .arg("x", json!(5)) .push(&db) @@ -1160,6 +1188,7 @@ mod dedicated_worker_tests { concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default( ), debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), + job_token_scopes: None, }) .arg("x", json!(5)) .push(&db) @@ -1222,6 +1251,7 @@ mod dedicated_worker_tests { concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default( ), debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), + job_token_scopes: None, }) .arg("x", json!(7)) .push(&db) @@ -1240,6 +1270,7 @@ mod dedicated_worker_tests { concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default( ), debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), + job_token_scopes: None, }) .workspace("test-workspace-2") .arg("x", json!(7)) @@ -1294,6 +1325,7 @@ mod dedicated_worker_tests { concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default( ), debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), + job_token_scopes: None, }) .arg("x", json!(5)) .push(&db) @@ -1338,6 +1370,7 @@ mod dedicated_worker_tests { concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default( ), debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), + job_token_scopes: None, }) .arg("x", json!(5)) .push(&db) @@ -1375,6 +1408,7 @@ mod dedicated_worker_tests { apply_preprocessor: false, labels: None, version: 3000000000000015, + job_token_scopes: None, }) .push(&db) .await; @@ -1412,6 +1446,7 @@ mod dedicated_worker_tests { apply_preprocessor: false, labels: None, version: 3000000000000019, + job_token_scopes: None, }) .push(&db) .await; @@ -1451,6 +1486,7 @@ mod dedicated_worker_tests { apply_preprocessor: false, labels: None, version: 3000000000000016, + job_token_scopes: None, }) .arg("x", json!(1)) .push(&db) @@ -1490,6 +1526,7 @@ mod dedicated_worker_tests { apply_preprocessor: false, labels: None, version: 3000000000000017, + job_token_scopes: None, }) .arg("x", json!(5)) .push(&db) @@ -1529,6 +1566,7 @@ mod dedicated_worker_tests { apply_preprocessor: false, labels: None, version: 3000000000000018, + job_token_scopes: None, }) .push(&db) .await; diff --git a/backend/tests/dependency_job_tag.rs b/backend/tests/dependency_job_tag.rs new file mode 100644 index 0000000000..58b320d2c1 --- /dev/null +++ b/backend/tests/dependency_job_tag.rs @@ -0,0 +1,126 @@ +//! Sets the process-wide `DEPENDENCY_JOB_TAG`, so it lives in its own test binary: any other +//! test queuing a dependency job in the same process would land on the overridden tag. + +use sqlx::{Pool, Postgres}; +use windmill_common::worker::DEPENDENCY_JOB_TAG; +use windmill_dep_map::scoped_dependency_map::DependencyDependent; + +#[sqlx::test(fixtures("base"))] +async fn relocks_go_to_the_configured_dependency_job_tag(db: Pool) -> anyhow::Result<()> { + DEPENDENCY_JOB_TAG.store(std::sync::Arc::new(Some("deps-$workspace".to_string()))); + + for (path, hash, tag) in [ + ("f/tags/tagged", 7_100_001_i64, Some("default-$workspace")), + ("f/tags/plain", 7_100_002_i64, None), + ] { + sqlx::query( + "INSERT INTO script(workspace_id, created_by, content, schema, summary, description, path, hash, language, tag) + VALUES ('test-workspace', 'test-user', 'def main(): pass', '{}', '', '', $1, $2, 'python3', $3)", + ) + .bind(path) + .bind(hash) + .bind(tag) + .execute(&db) + .await?; + } + let importers = ["f/tags/tagged", "f/tags/plain"] + .map(|p| DependencyDependent { + importer_path: p.to_string(), + importer_kind: "script".to_string(), + importer_node_ids: None, + }) + .to_vec(); + windmill_dep_map::trigger_dependents::trigger_dependents_to_recompute_dependencies( + "test-workspace", + importers, + None, + None, + "test@windmill.dev", + "test-user", + "u/test-user", + &db, + vec![], + ) + .await?; + + let tags: Vec<(String, String)> = sqlx::query_as( + "SELECT runnable_path, tag FROM v2_job WHERE kind = 'dependencies' ORDER BY runnable_path", + ) + .fetch_all(&db) + .await?; + assert_eq!( + tags, + vec![ + ( + "f/tags/plain".to_string(), + "deps-test-workspace".to_string() + ), + ( + "f/tags/tagged".to_string(), + "deps-test-workspace".to_string() + ), + ] + ); + + // A binary prebuild given `auto_build_binary_tag` keeps it; an untagged one follows its + // dependency job. + assert_eq!( + push_binary_prebuild(&db, Some("build-pool")).await?, + "build-pool" + ); + assert_eq!( + push_binary_prebuild(&db, None).await?, + "deps-test-workspace" + ); + Ok(()) +} + +async fn push_binary_prebuild(db: &Pool, tag: Option<&str>) -> anyhow::Result { + let mut args = std::collections::HashMap::new(); + args.insert( + "build_binary_only".to_string(), + windmill_common::worker::to_raw_value(&true), + ); + let (build_id, tx) = windmill_queue::push( + db, + windmill_queue::PushIsolationLevel::IsolatedRoot(db.clone()), + "test-workspace", + windmill_common::jobs::JobPayload::BuildBinary { + path: "f/tags/plain".to_string(), + hash: windmill_common::scripts::ScriptHash(7_100_002), + language: windmill_common::scripts::ScriptLang::Go, + }, + windmill_queue::PushArgs::from(&args), + "test-user", + "test@windmill.dev", + "u/test-user".to_string(), + None, + None, + None, + None, + None, + None, + None, + None, + false, + false, + None, + true, + tag.map(str::to_string), + None, + None, + None, + None, + false, + None, + None, + None, + None, + ) + .await?; + tx.commit().await?; + Ok(sqlx::query_scalar("SELECT tag FROM v2_job WHERE id = $1") + .bind(build_id) + .fetch_one(db) + .await?) +} diff --git a/backend/tests/dependency_map.rs b/backend/tests/dependency_map.rs index 59895b7a3a..dad6c86aa3 100644 --- a/backend/tests/dependency_map.rs +++ b/backend/tests/dependency_map.rs @@ -452,6 +452,7 @@ def main(): ws_error_handler_muted: None, labels: None, skip_draft_deletion: None, + job_token_scopes: None, }) .send() .await diff --git a/backend/tests/error_handler.rs b/backend/tests/error_handler.rs index 5d5b7a828b..4f58685221 100644 --- a/backend/tests/error_handler.rs +++ b/backend/tests/error_handler.rs @@ -161,6 +161,7 @@ export async function main(path: string, email: string, job_id: string, is_flow: concurrency_settings: ConcurrencySettings::default(), debouncing_settings: DebouncingSettings::default(), labels: None, + job_token_scopes: None, }) .run_until_complete(&db, false, server.addr.port()) .await; @@ -285,6 +286,7 @@ async fn test_error_handler_muted_on_script(db: Pool) -> anyhow::Resul concurrency_settings: ConcurrencySettings::default(), debouncing_settings: DebouncingSettings::default(), labels: None, + job_token_scopes: None, }) .run_until_complete(&db, false, server.addr.port()) .await; @@ -381,6 +383,7 @@ async fn test_error_handler_not_triggered_on_success(db: Pool) -> anyh concurrency_settings: ConcurrencySettings::default(), debouncing_settings: DebouncingSettings::default(), labels: None, + job_token_scopes: None, }) .run_until_complete(&db, false, server.addr.port()) .await; diff --git a/backend/tests/fixtures/jobs_read_auth.sql b/backend/tests/fixtures/jobs_read_auth.sql index 980d68b711..e32b777a74 100644 --- a/backend/tests/fixtures/jobs_read_auth.sql +++ b/backend/tests/fixtures/jobs_read_auth.sql @@ -29,6 +29,13 @@ INSERT INTO token(token_hash, token_prefix, token, email, label, super_admin, sc ARRAY['jobs:run:flows:f/shared/flow1'] ); +-- A path-scoped read token for test-user-2: it reads the runs of `f/shared/flow1` only. +INSERT INTO token(token_hash, token_prefix, token, email, label, super_admin, scopes) VALUES ( + encode(sha256('READ_SCOPED_TOKEN'::bytea), 'hex'), 'READ_SCOP', 'READ_SCOPED_TOKEN', + 'test2@windmill.dev', 'flow history token', false, + ARRAY['jobs:read:f/shared/flow1'] +); + -- Same, scoped to a script. The two jobs below both run through a `singlestepflow` -- wrapper (native retry / scheduled runs produce these) — one wrapping a script, one -- wrapping a flow — so the confinement has to project each onto the runnable it wraps diff --git a/backend/tests/flow_engine_parity.rs b/backend/tests/flow_engine_parity.rs index 7ea221c2e8..3806bb2886 100644 --- a/backend/tests/flow_engine_parity.rs +++ b/backend/tests/flow_engine_parity.rs @@ -50,6 +50,7 @@ fn flow_module(id: &str, value: FlowModuleValue) -> FlowModule { apply_preprocessor: None, pass_flow_input_directly: None, debouncing: None, + job_token_scopes: None, } } @@ -2974,6 +2975,7 @@ async fn test_flow_env_marks_sub_flows_only_without_ancestor_env( apply_preprocessor: false, version, labels: None, + job_token_scopes: None, }) .run_until_complete(&db, false, port) .await; diff --git a/backend/tests/job_payload.rs b/backend/tests/job_payload.rs index 1a973f6fe6..0038effc78 100644 --- a/backend/tests/job_payload.rs +++ b/backend/tests/job_payload.rs @@ -57,6 +57,7 @@ mod job_payload { language: ScriptLang::Deno, priority: None, apply_preprocessor: false, + job_token_scopes: None, }) .arg("world", json!("foo")) .run_until_complete(&db, false, port) @@ -92,6 +93,7 @@ mod job_payload { debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), labels: None, + job_token_scopes: None, }) .run_until_complete_with(db, false, port, |id| async move { let job = sqlx::query!("SELECT preprocessed FROM v2_job WHERE id = $1", id) @@ -435,6 +437,7 @@ mod job_payload { apply_preprocessor: false, version: 1443253234253454, labels: None, + job_token_scopes: None, }) .run_until_complete(&db, false, port) .await @@ -485,6 +488,7 @@ mod job_payload { apply_preprocessor: true, version: 1443253234253456, labels: None, + job_token_scopes: None, }) .run_until_complete_with(db, false, port, |id| async move { let job = sqlx::query!("SELECT preprocessed FROM v2_job WHERE id = $1", id) @@ -556,6 +560,7 @@ mod job_payload { apply_preprocessor: true, version: 1443253234253454, labels: None, + job_token_scopes: None, }) .run_until_complete(&db, false, port) .await @@ -1150,6 +1155,7 @@ mod job_payload { apply_preprocessor: true, version: 1443253234253454, labels: None, + job_token_scopes: None, }) .run_until_complete(db, false, port) .await; @@ -1625,6 +1631,7 @@ mod job_payload { debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), labels: None, + job_token_scopes: None, }) .arg("foo", json!("hello")) .arg("bar", json!("world")) @@ -1676,6 +1683,7 @@ mod job_payload { debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), labels: None, + job_token_scopes: None, }) .arg("foo", json!("hello")) .arg("bar", json!("world")) @@ -1727,6 +1735,7 @@ mod job_payload { debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), labels: None, + job_token_scopes: None, }) .arg("foo", json!("hello")) .arg("bar", json!("world")) @@ -1778,6 +1787,7 @@ mod job_payload { debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), labels: None, + job_token_scopes: None, }) .arg("foo", json!("hello")) .arg("bar", json!("world")) @@ -1831,6 +1841,7 @@ mod job_payload { debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), labels: None, + job_token_scopes: None, }) .arg("foo", json!("hello")) .arg("bar", json!("world")) diff --git a/backend/tests/job_provenance.rs b/backend/tests/job_provenance.rs index 42ccf829e6..4eda4e5e17 100644 --- a/backend/tests/job_provenance.rs +++ b/backend/tests/job_provenance.rs @@ -150,3 +150,104 @@ async fn flow_steps_are_latest_only_under_a_current_unrestarted_flow(db: Pool) { + use serde_json::{json, Value}; + use windmill_common::job_provenance::RunAsType; + + let v: Value = + serde_json::from_str(include_str!("../../cli/test/fixtures/item_digest_vectors.json")) + .unwrap(); + let (script, flow) = (&v["script"], &v["flow"]); + + sqlx::query( + "INSERT INTO script (workspace_id, hash, path, content, lock, modules, language, kind, created_by, schema, summary, description) + VALUES ('test-workspace', 777, $1, $2, $3, $4, 'bun', 'script', 'test-user', '{}', '', '')", + ) + .bind(script["path"].as_str()) + .bind(script["content"].as_str()) + .bind(script["lock"].as_str()) + .bind(&script["modules"]) + .execute(&db) + .await + .unwrap(); + + // Version 889 holds step `a` by reference, which no checkout can reproduce. + let step = &flow["value"]["modules"][0]["value"]; + let mut by_ref = flow["value"].clone(); + by_ref["modules"][0]["value"] = json!({ + "type": "flowscript", "id": 999, "language": step["language"], + "input_transforms": step["input_transforms"], + }); + sqlx::query( + "INSERT INTO flow (workspace_id, path, summary, description, value, edited_by, versions) + VALUES ('test-workspace', $1, '', '', $2, 'test-user', ARRAY[888::bigint, 889::bigint])", + ) + .bind(flow["path"].as_str()) + .bind(&flow["value"]) + .execute(&db) + .await + .unwrap(); + for (id, value) in [(888, &flow["value"]), (889, &by_ref)] { + sqlx::query( + "INSERT INTO flow_version (id, workspace_id, path, value, schema, created_by) + VALUES ($1, 'test-workspace', $2, $3, '{}', 'test-user')", + ) + .bind(id as i64) + .bind(flow["path"].as_str()) + .bind(value) + .execute(&db) + .await + .unwrap(); + } + sqlx::query( + "INSERT INTO flow_node (id, workspace_id, path, code, lock, hash_v2) + VALUES (999, 'test-workspace', $1, $2, $3, 'h')", + ) + .bind(flow["path"].as_str()) + .bind(step["content"].as_str()) + .bind(step["lock"].as_str()) + .execute(&db) + .await + .unwrap(); + + sqlx::query( + "INSERT INTO v2_job (id, workspace_id, kind, runnable_path, runnable_id, parent_job, trigger, trigger_kind, tag, created_by, permissioned_as, permissioned_as_email) + VALUES + ('3bb0c0de-0000-4000-8000-000000000201', 'test-workspace', 'script', $1, 777, NULL, NULL, NULL, 'bun', 'test-user', 'g/all', 'group-all@windmill.dev'), + ('3bb0c0de-0000-4000-8000-000000000202', 'test-workspace', 'flow', $2, 888, NULL, 'f/digest/nightly', 'schedule', 'flow', 'test-user', 'u/test-user', 'test@windmill.dev'), + ('3bb0c0de-0000-4000-8000-000000000203', 'test-workspace', 'flowscript', $2 || '/a', 999, '3bb0c0de-0000-4000-8000-000000000202', NULL, NULL, 'gpu', 'test-user', 'u/test-user', 'test@windmill.dev'), + ('3bb0c0de-0000-4000-8000-000000000204', 'test-workspace', 'flow', $2, 889, NULL, NULL, NULL, 'flow', 'test-user', 'u/test-user', 'test@windmill.dev')", + ) + .bind(script["path"].as_str()) + .bind(flow["path"].as_str()) + .execute(&db) + .await + .unwrap(); + for q in [ + "INSERT INTO worker_ping (worker, worker_instance, worker_group) VALUES ('wk-gpu', 'wk', 'gpu-group')", + "INSERT INTO v2_job_queue (id, workspace_id, scheduled_for, running, worker, tag) + VALUES ('3bb0c0de-0000-4000-8000-000000000203', 'test-workspace', now(), true, 'wk-gpu', 'gpu')", + ] { + sqlx::query(q).execute(&db).await.unwrap(); + } + + let s = provenance(&db, "3bb0c0de-0000-4000-8000-000000000201").await; + assert_eq!(s.digest.as_deref(), script["digest"].as_str()); + assert_eq!(s.root_digest, s.digest); + assert_eq!(s.run_as_type, RunAsType::Group); + assert_eq!(s.worker_group, None); + + let step = provenance(&db, "3bb0c0de-0000-4000-8000-000000000203").await; + assert_eq!(step.digest.as_deref(), flow["steps"]["a"].as_str()); + assert_eq!(step.root_digest.as_deref(), flow["digest"].as_str()); + assert_eq!(step.root_trigger.as_deref(), Some("f/digest/nightly")); + assert_eq!(step.tag, "gpu"); + assert_eq!(step.worker_group.as_deref(), Some("gpu-group")); + assert_eq!(step.run_as_type, RunAsType::User); + + assert_eq!(provenance(&db, "3bb0c0de-0000-4000-8000-000000000204").await.digest, None); +} diff --git a/backend/tests/job_token_scopes.rs b/backend/tests/job_token_scopes.rs new file mode 100644 index 0000000000..6af694bff0 --- /dev/null +++ b/backend/tests/job_token_scopes.rs @@ -0,0 +1,414 @@ +//! A job token restricted by `job_token_scopes` reaches only what its scopes allow, plus the +//! runtime routes about its own job, and no job it creates holds a wider token than it. + +use reqwest::StatusCode; +use serde_json::json; +use sqlx::{Pool, Postgres}; +use uuid::Uuid; +use windmill_common::jobs::JobPayload; +use windmill_test_utils::*; + +const OIDC_JOB: &str = "b0000000-0000-0000-0000-000000000001"; +const RUN_JOB: &str = "b0000000-0000-0000-0000-000000000002"; +const FLOW_JOB: &str = "b0000000-0000-0000-0000-000000000003"; + +async fn insert_job( + db: &Pool, + id: &str, + parent: Option<&str>, + scopes: &[&str], +) -> anyhow::Result<()> { + insert_job_with(db, id, parent, Some(scopes)).await +} + +async fn insert_job_with( + db: &Pool, + id: &str, + parent: Option<&str>, + scopes: Option<&[&str]>, +) -> anyhow::Result<()> { + let id = Uuid::parse_str(id)?; + sqlx::query( + "INSERT INTO v2_job (id, workspace_id, created_by, permissioned_as, permissioned_as_email, + kind, script_lang, runnable_path, tag, parent_job) + VALUES ($1, 'test-workspace', 'test-user-3', 'u/test-user-3', 'test3@windmill.dev', + 'script', 'deno', 'u/test-user-3/agent', 'deno', $2)", + ) + .bind(id) + .bind(parent.map(Uuid::parse_str).transpose()?) + .execute(db) + .await?; + sqlx::query( + "INSERT INTO job_perms (job_id, email, username, is_admin, is_operator, folders, groups, + workspace_id, job_token_scopes) + VALUES ($1, 'test3@windmill.dev', 'test-user-3', false, false, '{}', '{}', + 'test-workspace', $2)", + ) + .bind(id) + .bind(scopes) + .execute(db) + .await?; + Ok(()) +} + +async fn job_token(db: &Pool, id: &str) -> anyhow::Result { + Ok(windmill_common::auth::create_token_for_owner( + db, + "test-workspace", + "u/test-user-3", + "ephemeral-script", + 300, + "test3@windmill.dev", + &Uuid::parse_str(id)?, + None, + None, + ) + .await?) +} + +async fn insert_script( + db: &Pool, + path: &str, + hash: i64, + scopes: Option<&[&str]>, +) -> anyhow::Result<()> { + sqlx::query( + "INSERT INTO script (workspace_id, created_by, content, schema, summary, description, + path, hash, language, lock, kind, job_token_scopes) + VALUES ('test-workspace', 'test-user-3', 'export function main() {}', '{}', '', '', + $1, $2, 'deno', '', 'script', $3)", + ) + .bind(path) + .bind(hash) + .bind(scopes) + .execute(db) + .await?; + Ok(()) +} + +#[sqlx::test(fixtures("base"))] +async fn test_restricted_job_token_is_confined(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + insert_script(&db, "u/test-user-3/open", 515151, None).await?; + insert_script( + &db, + "u/test-user-3/oidc_only", + 525252, + Some(&["oidc:write"]), + ) + .await?; + insert_job(&db, FLOW_JOB, None, &["oidc:write"]).await?; + insert_job(&db, OIDC_JOB, Some(FLOW_JOB), &["oidc:write"]).await?; + insert_job(&db, RUN_JOB, None, &["jobs:run"]).await?; + + let server = ApiServer::start(db.clone()).await?; + set_jwt_secret().await; + let base = format!( + "http://localhost:{}/api/w/test-workspace", + server.addr.port() + ); + let client = reqwest::Client::new(); + let oidc_token = job_token(&db, OIDC_JOB).await?; + + let refused = [ + client.get(format!("{base}/variables/get_value/u/test-user-3/secret")), + client + .post(format!("{base}/scripts/create")) + .json(&json!({})), + client + .post(format!("{base}/schedules/create")) + .json(&json!({})), + client + .post(format!("{base}/jobs/run/p/u/test-user-3/open")) + .json(&json!({})), + ]; + for request in refused { + let resp = request.bearer_auth(&oidc_token).send().await?; + assert_eq!( + resp.status(), + StatusCode::FORBIDDEN, + "{}", + resp.text().await? + ); + } + let own = client + .get(format!("{base}/jobs_u/get_root_job_id/{OIDC_JOB}")) + .bearer_auth(&oidc_token) + .send() + .await?; + assert_eq!(own.status(), StatusCode::OK, "{}", own.text().await?); + // The orchestrator reads a flow's step results with the token of the step that just ran. + for (job, refused) in [(FLOW_JOB, false), (RUN_JOB, true)] { + let resp = client + .get(format!("{base}/jobs/result_by_id/{job}/a")) + .bearer_auth(&oidc_token) + .send() + .await?; + assert_eq!(resp.status() == StatusCode::FORBIDDEN, refused, "{job}"); + } + // Its progress reaches only the flow it runs in. + for (flow, refused) in [(FLOW_JOB, false), (RUN_JOB, true)] { + let resp = client + .post(format!("{base}/job_metrics/set_progress/{OIDC_JOB}")) + .bearer_auth(&oidc_token) + .json(&json!({ "percent": 50, "flow_job_id": flow })) + .send() + .await?; + assert_eq!(resp.status() == StatusCode::FORBIDDEN, refused, "{flow}"); + } + + // Imported queue rows bypass push, so a restricted token cannot import, even an admin's. + let admin_job = Uuid::parse_str("b0000000-0000-0000-0000-000000000004")?; + sqlx::query( + "INSERT INTO v2_job (id, workspace_id, created_by, permissioned_as, permissioned_as_email, + kind, script_lang, runnable_path, tag) + VALUES ($1, 'test-workspace', 'test-user', 'u/test-user', 'test@windmill.dev', + 'script', 'deno', 'u/test-user/agent', 'deno')", + ) + .bind(admin_job) + .execute(&db) + .await?; + sqlx::query( + "INSERT INTO job_perms (job_id, email, username, is_admin, is_operator, folders, groups, + workspace_id, job_token_scopes) + VALUES ($1, 'test@windmill.dev', 'test-user', true, false, '{}', '{}', 'test-workspace', + '{jobs:write,jobs:run}')", + ) + .bind(admin_job) + .execute(&db) + .await?; + let admin_token = windmill_common::auth::create_token_for_owner( + &db, + "test-workspace", + "u/test-user", + "ephemeral-script", + 300, + "test@windmill.dev", + &admin_job, + None, + None, + ) + .await?; + let resp = client + .post(format!("{base}/jobs/queue/import")) + .bearer_auth(&admin_token) + .json(&json!([])) + .send() + .await?; + assert_eq!(resp.status(), StatusCode::FORBIDDEN); + assert!(resp.text().await?.contains("cannot import")); + // Nor can it, even an admin's, place a child in an unrelated run: the flow-run routes + // trust that lineage. + let resp = client + .post(format!( + "{base}/jobs/run/p/u/test-user-3/open?root_job={RUN_JOB}" + )) + .bearer_auth(&admin_token) + .json(&json!({})) + .send() + .await?; + assert_eq!(resp.status(), StatusCode::CREATED, "{}", resp.text().await?); + let child = Uuid::parse_str(&resp.text().await?)?; + let claimed: bool = sqlx::query_scalar( + "SELECT $2 IN (parent_job, root_job, flow_innermost_root_job) IS TRUE FROM v2_job + WHERE id = $1", + ) + .bind(child) + .bind(Uuid::parse_str(RUN_JOB)?) + .fetch_one(&db) + .await?; + assert!(!claimed); + + // A job it starts is capped at its own scopes, intersected with the target's setting. + let run_token = job_token(&db, RUN_JOB).await?; + for (path, expected) in [ + ("open", vec!["jobs:run".to_string()]), + ("oidc_only", vec![]), + ] { + let resp = client + .post(format!("{base}/jobs/run/p/u/test-user-3/{path}")) + .bearer_auth(&run_token) + .json(&json!({})) + .send() + .await?; + assert_eq!(resp.status(), StatusCode::CREATED); + let child = Uuid::parse_str(&resp.text().await?)?; + let scopes: Option> = + sqlx::query_scalar("SELECT job_token_scopes FROM job_perms WHERE job_id = $1") + .bind(child) + .fetch_one(&db) + .await?; + assert_eq!(scopes, Some(expected), "child of {path}"); + } + Ok(()) +} + +#[sqlx::test(fixtures("base"))] +async fn test_flow_steps_inherit_the_flow_restriction(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let value = json!({ "modules": [ + { "id": "a", "value": { "type": "identity" }, + "job_token_scopes": ["oidc:write", "variables:read"] }, + { "id": "b", "value": { "type": "identity" } }, + ] }); + sqlx::query( + "INSERT INTO flow (workspace_id, path, summary, description, value, edited_by, edited_at, + schema, extra_perms, versions) + VALUES ('test-workspace', 'u/test-user/agent_flow', '', '', $1, 'test-user', now(), '{}', + '{}', '{7171}')", + ) + .bind(&value) + .execute(&db) + .await?; + sqlx::query( + "INSERT INTO flow_version (id, workspace_id, path, value, schema, created_by) + VALUES (7171, 'test-workspace', 'u/test-user/agent_flow', $1, '{}', 'test-user')", + ) + .bind(&value) + .execute(&db) + .await?; + + let server = ApiServer::start(db.clone()).await?; + let scopes = |v: &[&str]| Some(v.iter().map(|s| s.to_string()).collect::>()); + // A step's own setting narrows the flow's restriction and never widens it; in an + // unrestricted flow it restricts that step alone. + for (flow_scopes, expected_a, expected_b) in [ + (scopes(&["oidc:write"]), scopes(&["oidc:write"]), scopes(&["oidc:write"])), + (None, scopes(&["oidc:write", "variables:read"]), None), + ] { + let flow = RunJob::from(JobPayload::Flow { + path: "u/test-user/agent_flow".to_string(), + dedicated_worker: None, + apply_preprocessor: false, + version: 7171, + labels: None, + job_token_scopes: flow_scopes.clone(), + }) + .run_until_complete(&db, false, server.addr.port()) + .await; + + let steps: Vec<(String, Option>)> = sqlx::query_as( + "SELECT j.flow_step_id, p.job_token_scopes FROM v2_job j + JOIN job_perms p ON p.job_id = j.id + WHERE j.parent_job = $1 ORDER BY j.flow_step_id", + ) + .bind(flow.id) + .fetch_all(&db) + .await?; + assert_eq!( + steps, + vec![("a".to_string(), expected_a), ("b".to_string(), expected_b)], + "flow scopes {flow_scopes:?}" + ); + } + Ok(()) +} + +#[sqlx::test(fixtures("base"))] +async fn test_deploy_without_the_field_keeps_the_restriction( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let base = format!("http://localhost:{}/api/w/test-workspace", server.addr.port()); + let client = reqwest::Client::new(); + let script = |path: &str, scopes: Option| { + let mut body = json!({ + "path": path, "summary": "", "description": "", "content": "echo 42", "language": "bash", + }); + if let Some(scopes) = scopes { + body["job_token_scopes"] = scopes; + } + body + }; + let stored = |path: &'static str| { + let db = db.clone(); + async move { + sqlx::query_scalar::<_, Option>>( + "SELECT job_token_scopes FROM script WHERE path = $1 ORDER BY created_at DESC LIMIT 1", + ) + .bind(path) + .fetch_one(&db) + .await + } + }; + + let resp = client + .post(format!("{base}/scripts/create")) + .bearer_auth("SECRET_TOKEN") + .json(&script("u/test-user/agent", Some(json!(["oidc:write"])))) + .send() + .await?; + assert_eq!(resp.status(), StatusCode::CREATED, "{}", resp.text().await?); + + // A rename by a client that does not know the field carries the restriction along. + let resp = client + .post(format!("{base}/scripts/update/u/test-user/agent")) + .bearer_auth("SECRET_TOKEN") + .json(&script("u/test-user/renamed", None)) + .send() + .await?; + assert!(resp.status().is_success(), "{}", resp.text().await?); + assert_eq!(stored("u/test-user/renamed").await?, Some(vec!["oidc:write".to_string()])); + + // An explicit null clears it. + let resp = client + .post(format!("{base}/scripts/update/u/test-user/renamed")) + .bearer_auth("SECRET_TOKEN") + .json(&script("u/test-user/renamed", Some(serde_json::Value::Null))) + .send() + .await?; + assert!(resp.status().is_success(), "{}", resp.text().await?); + assert_eq!(stored("u/test-user/renamed").await?, None); + Ok(()) +} + +/// The OIDC `job_token_scopes` claim states the scopes of the token that asked for it. +#[cfg(all(feature = "enterprise", feature = "private", feature = "openidconnect"))] +#[sqlx::test(fixtures("base"))] +async fn test_oidc_claim_carries_the_presented_scopes(db: Pool) -> anyhow::Result<()> { + use base64::Engine; + + initialize_tracing().await; + const OPEN_JOB: &str = "b0000000-0000-0000-0000-000000000005"; + insert_script(&db, "u/test-user-3/agent", 535353, None).await?; + insert_job(&db, OIDC_JOB, None, &["oidc:write"]).await?; + insert_job_with(&db, OPEN_JOB, None, None).await?; + sqlx::query("UPDATE v2_job SET runnable_id = 535353 WHERE id = ANY($1)") + .bind([Uuid::parse_str(OIDC_JOB)?, Uuid::parse_str(OPEN_JOB)?]) + .execute(&db) + .await?; + sqlx::query( + "INSERT INTO v2_job_queue (id, workspace_id, running, scheduled_for, tag) + SELECT id, workspace_id, true, now(), tag FROM v2_job WHERE id = ANY($1)", + ) + .bind([Uuid::parse_str(OIDC_JOB)?, Uuid::parse_str(OPEN_JOB)?]) + .execute(&db) + .await?; + + let server = ApiServer::start(db.clone()).await?; + set_jwt_secret().await; + // The issuer of the minted token. + windmill_common::BASE_URL.store(std::sync::Arc::new("http://localhost".to_string())); + let base = format!( + "http://localhost:{}/api/w/test-workspace", + server.addr.port() + ); + let client = reqwest::Client::new(); + for (job, expected) in [(OIDC_JOB, json!(["oidc:write"])), (OPEN_JOB, json!(null))] { + let resp = client + .post(format!("{base}/oidc/token/test-audience")) + .bearer_auth(job_token(&db, job).await?) + .send() + .await?; + let status = resp.status(); + let id_token = resp.text().await?; + assert_eq!(status, StatusCode::OK, "{id_token}"); + let payload = id_token.split('.').nth(1).expect("a JWT payload"); + let claims: serde_json::Value = serde_json::from_slice( + &base64::engine::general_purpose::URL_SAFE_NO_PAD.decode(payload)?, + )?; + assert_eq!(claims["job_token_scopes"], expected, "{job}"); + } + Ok(()) +} diff --git a/backend/tests/jobs_read_auth.rs b/backend/tests/jobs_read_auth.rs index fd2969a18f..6a101200a1 100644 --- a/backend/tests/jobs_read_auth.rs +++ b/backend/tests/jobs_read_auth.rs @@ -564,6 +564,56 @@ async fn test_single_job_read_authorization(db: Pool) -> anyhow::Resul "run-scoped token must not enumerate jobs (got {status}): {body}" ); + // ---- PATH-SCOPED READ TOKEN: READ_SCOPED_TOKEN is test-user-2's + // `jobs:read:f/shared/flow1`. It reads that flow's runs and the steps beneath + // them, and nothing else its owner created. + for (path, expected) in [ + ( + format!("completed/get_result/{FLOW_JOB}"), + r#""flow": "done""#, + ), + ( + format!("completed/get_result/{STEP_JOB}"), + "STEP_RESULT_INHERITED", + ), + ] { + let (status, body) = get(&base, &path, Some("READ_SCOPED_TOKEN")).await; + assert!( + status.is_success() && body.contains(expected), + "read-scoped token must read a run of its flow ({path}, got {status}): {body}" + ); + } + for path in [ + format!("completed/get_result/{VICTIM}"), + format!("get/{VICTIM}"), + format!("flow/approval_info/{VICTIM}"), + ] { + let (status, body) = get(&base, &path, Some("READ_SCOPED_TOKEN")).await; + assert_eq!( + status, + reqwest::StatusCode::NOT_FOUND, + "read-scoped token must not read a job outside its paths ({path}, got {status}): {body}" + ); + } + // The lists answer with the top-level runs at its paths only, where the same user's + // unscoped token sees the other runs too. + for list in ["list", "completed/list"] { + let (status, body) = get(&authed_base, list, Some("READ_SCOPED_TOKEN")).await; + assert!(status.is_success(), "{list} (got {status}): {body}"); + let jobs: Vec = serde_json::from_str(&body)?; + let mut ids: Vec<&str> = jobs.iter().filter_map(|j| j["id"].as_str()).collect(); + ids.sort(); + assert_eq!(ids, [WRAPPED_FLOW_JOB, FLOW_JOB], "{list}: {body}"); + } + let (_, body) = get(&authed_base, "list", Some("SECRET_TOKEN_2")).await; + assert!(body.contains(VICTIM), "unscoped list: {body}"); + let (status, body) = get(&authed_base, "completed/count", Some("READ_SCOPED_TOKEN")).await; + assert_eq!( + status, + reqwest::StatusCode::FORBIDDEN, + "read-scoped token must not count the workspace's jobs (got {status}): {body}" + ); + // ---- APP EMBED TOKEN: cancellation confined to the app's own jobs. The token // may cancel a job it launched (created_by == viewer), but `cancel_job_api` // denies (NotFound) a job created by someone else, even one the (admin) diff --git a/backend/tests/list_jobs.rs b/backend/tests/list_jobs.rs index 29a49c1610..d33618d048 100644 --- a/backend/tests/list_jobs.rs +++ b/backend/tests/list_jobs.rs @@ -957,6 +957,7 @@ async fn test_job_labels_propagated_at_push_time(db: Pool) -> anyhow:: .into(), debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), labels: Some(vec!["prod".to_string(), "deploy".to_string()]), + job_token_scopes: None, }) .push(&db) .await; @@ -1060,6 +1061,7 @@ async fn test_job_label_filter(db: Pool) -> anyhow::Result<()> { .into(), debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), labels: Some(vec!["prod".to_string()]), + job_token_scopes: None, }) .push(&db) .await; @@ -1077,6 +1079,7 @@ async fn test_job_label_filter(db: Pool) -> anyhow::Result<()> { .into(), debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), labels: Some(vec!["staging".to_string()]), + job_token_scopes: None, }) .push(&db) .await; diff --git a/backend/tests/nativets_stress.rs b/backend/tests/nativets_stress.rs index 21d3911778..d1e371ef81 100644 --- a/backend/tests/nativets_stress.rs +++ b/backend/tests/nativets_stress.rs @@ -191,6 +191,7 @@ async fn push_job(db: &Pool, content: &str, args: &serde_json::Value) None, None, None, + None, ) .await .expect("push must succeed"); diff --git a/backend/tests/preview_native_tag.rs b/backend/tests/preview_native_tag.rs index eba686396a..39f985cefe 100644 --- a/backend/tests/preview_native_tag.rs +++ b/backend/tests/preview_native_tag.rs @@ -76,6 +76,7 @@ async fn push_preview_and_get_row( None, None, None, + None, ) .await .expect("push must succeed"); diff --git a/backend/tests/python_jobs.rs b/backend/tests/python_jobs.rs index 9616df267a..4236599aba 100644 --- a/backend/tests/python_jobs.rs +++ b/backend/tests/python_jobs.rs @@ -496,6 +496,28 @@ async def main(x: int): ); } + // `hasattr` does not swallow the KeyError this `__getattr__` raises; released + // `wmill` clients ship an `S3Object` built this way. + #[test] + fn test_python_execd_result_with_keyerror_getattr() { + let script = r#" +class AttrDict(dict): + def __getattr__(self, attr): + return self[attr] + +def main(x: str): + return AttrDict(s3=x) +"#; + let results = run_py_raw_protocol_test( + &[("f/test/attr_dict", script)], + vec![ProtocolCmd::Execd { args: serde_json::json!({"x": "a/b.txt"}) }], + ); + assert_eq!( + results, + vec![DedicatedWorkerResult::Success(serde_json::json!({"s3": "a/b.txt"}))] + ); + } + // ==================== Argument Transformation Tests ==================== #[test] @@ -1278,6 +1300,7 @@ async fn test_python_wac_v2_with_preprocessor(db: Pool) -> anyhow::Res debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), labels: None, + job_token_scopes: None, }) .arg("who", json!("alice")) .arg("count", json!(7)) diff --git a/backend/tests/restart_at_step_api.rs b/backend/tests/restart_at_step_api.rs index c5ab703f46..2b57de2c64 100644 --- a/backend/tests/restart_at_step_api.rs +++ b/backend/tests/restart_at_step_api.rs @@ -41,6 +41,7 @@ async fn test_api_restart_at_step_nested_happy(db: Pool) -> anyhow::Re apply_preprocessor: true, version: 1443253234253454, labels: None, + job_token_scopes: None, }) .run_until_complete(&db, false, port) .await; @@ -85,6 +86,7 @@ async fn test_api_restart_at_step_top_level_happy(db: Pool) -> anyhow: apply_preprocessor: true, version: 1443253234253454, labels: None, + job_token_scopes: None, }) .run_until_complete(&db, false, port) .await; @@ -118,6 +120,7 @@ async fn test_api_restart_at_step_rejects_unknown_step(db: Pool) -> an apply_preprocessor: true, version: 1443253234253454, labels: None, + job_token_scopes: None, }) .run_until_complete(&db, false, port) .await; @@ -161,6 +164,7 @@ async fn test_api_restart_at_step_rejects_out_of_range_iteration( apply_preprocessor: true, version: 1443253234253454, labels: None, + job_token_scopes: None, }) .run_until_complete(&db, false, port) .await; diff --git a/backend/tests/restart_flow_version_scope.rs b/backend/tests/restart_flow_version_scope.rs index 82e8f8e4e7..551e8c3619 100644 --- a/backend/tests/restart_flow_version_scope.rs +++ b/backend/tests/restart_flow_version_scope.rs @@ -53,6 +53,7 @@ async fn push_restart( None, None, None, + None, ) .await?; tx.commit().await?; diff --git a/backend/tests/success_handler.rs b/backend/tests/success_handler.rs index fe6127c27e..8fc4182d9e 100644 --- a/backend/tests/success_handler.rs +++ b/backend/tests/success_handler.rs @@ -181,6 +181,7 @@ export async function main(path: string, email: string, job_id: string, is_flow: concurrency_settings: ConcurrencySettings::default(), debouncing_settings: DebouncingSettings::default(), labels: None, + job_token_scopes: None, }) .run_until_complete(&db, false, server.addr.port()) .await; diff --git a/backend/tests/v2_job_delete_orphans.rs b/backend/tests/v2_job_delete_orphans.rs index 2b11deab77..45029f024d 100644 --- a/backend/tests/v2_job_delete_orphans.rs +++ b/backend/tests/v2_job_delete_orphans.rs @@ -205,6 +205,75 @@ async fn test_delete_jobs_removes_a_conversation_once_its_last_message_goes( Ok(()) } +/// The agent step completes, and so expires, before the flow run that holds the question: its +/// job going must leave the answer in place, and the run going must take the whole turn. +#[sqlx::test(fixtures("base"))] +async fn test_delete_jobs_removes_a_turn_with_its_flow_run( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + + let flow_job = Uuid::new_v4(); + let step_job = Uuid::new_v4(); + insert_job(&db, WS, flow_job).await?; + insert_job(&db, WS, step_job).await?; + + let conv_id = Uuid::new_v4(); + sqlx::query( + "INSERT INTO flow_conversation (id, workspace_id, flow_path, created_by) + VALUES ($1, $2, 'f/flow', 'test-user')", + ) + .bind(conv_id) + .bind(WS) + .execute(&db) + .await?; + let mut tx = db.begin().await?; + for (job_id, message_type) in [ + ( + flow_job, + windmill_common::flow_conversations::MessageType::User, + ), + ( + step_job, + windmill_common::flow_conversations::MessageType::Assistant, + ), + ] { + windmill_common::flow_conversations::add_message_to_conversation_tx( + &mut tx, + conv_id, + Some(job_id), + flow_job, + "hi", + message_type, + None, + true, + None, + ) + .await?; + } + tx.commit().await?; + + let messages = "SELECT count(*) FROM flow_conversation_message WHERE conversation_id = $1"; + let mut conn = db.acquire().await?; + windmill_common::jobs::delete_jobs(&mut conn, &[step_job]).await?; + drop(conn); + assert_eq!( + count(&db, messages, conv_id).await?, + 2, + "the step's job going must keep the turn" + ); + + let mut conn = db.acquire().await?; + windmill_common::jobs::delete_jobs(&mut conn, &[flow_job]).await?; + drop(conn); + assert_eq!( + count(&db, messages, conv_id).await?, + 0, + "the run going must take the turn" + ); + Ok(()) +} + /// Turns that start while retention is collecting their conversation must land, not fail: /// the conversation lookup locks the row, so each turn waits for the collector's commit, /// finds the conversation gone, and creates it again — the first insert wins and the other @@ -265,6 +334,7 @@ async fn test_new_turns_wait_for_conversation_cleanup_and_recreate( &mut tx, conv_id, Some(new_job), + new_job, "hi again", windmill_common::flow_conversations::MessageType::User, None, diff --git a/backend/tests/worker.rs b/backend/tests/worker.rs index e00ce1172b..f420dd2a12 100644 --- a/backend/tests/worker.rs +++ b/backend/tests/worker.rs @@ -212,6 +212,7 @@ async fn test_deno_flow(db: Pool) -> anyhow::Result<()> { apply_preprocessor: None, pass_flow_input_directly: None, debouncing: None, + job_token_scopes: None, }, FlowModule { id: "b".to_string(), @@ -260,6 +261,7 @@ async fn test_deno_flow(db: Pool) -> anyhow::Result<()> { apply_preprocessor: None, pass_flow_input_directly: None, debouncing: None, + job_token_scopes: None, }], modules_node: None, } @@ -282,6 +284,7 @@ async fn test_deno_flow(db: Pool) -> anyhow::Result<()> { apply_preprocessor: None, pass_flow_input_directly: None, debouncing: None, + job_token_scopes: None, }, ], same_worker: false, @@ -398,6 +401,7 @@ async fn test_deno_flow_same_worker(db: Pool) -> anyhow::Result<()> { apply_preprocessor: None, pass_flow_input_directly: None, debouncing: None, + job_token_scopes: None, }, FlowModule { id: "b".to_string(), @@ -455,6 +459,7 @@ async fn test_deno_flow_same_worker(db: Pool) -> anyhow::Result<()> { apply_preprocessor: None, pass_flow_input_directly: None, debouncing: None, + job_token_scopes: None, }, FlowModule { id: "e".to_string(), @@ -498,6 +503,7 @@ async fn test_deno_flow_same_worker(db: Pool) -> anyhow::Result<()> { apply_preprocessor: None, pass_flow_input_directly: None, debouncing: None, + job_token_scopes: None, }, ], modules_node: None, @@ -520,6 +526,7 @@ async fn test_deno_flow_same_worker(db: Pool) -> anyhow::Result<()> { apply_preprocessor: None, pass_flow_input_directly: None, debouncing: None, + job_token_scopes: None, }, FlowModule { id: "c".to_string(), @@ -569,6 +576,7 @@ async fn test_deno_flow_same_worker(db: Pool) -> anyhow::Result<()> { apply_preprocessor: None, pass_flow_input_directly: None, debouncing: None, + job_token_scopes: None, }, ], same_worker: true, @@ -1723,6 +1731,286 @@ async fn test_postgresql_cached_connection_resets_session( Ok(()) } +/// The idle cached connection must not hold the server slot a job's own +/// connection needs. A role limited to one connection stands in for a +/// session-mode pooler with one slot, where the second job would wait forever +/// instead of failing. +#[sqlx::test(fixtures("base"))] +#[serial(pg_cache)] +async fn test_postgresql_cached_connection_released_for_other_key( + db: Pool, +) -> anyhow::Result<()> { + use windmill_worker::pg_executor::clear_pg_cache; + + initialize_tracing().await; + clear_pg_cache().await; + + sqlx::query( + "DO $$ BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'wm_pg_cache_one_conn') THEN + CREATE ROLE wm_pg_cache_one_conn LOGIN PASSWORD 'changeme' CONNECTION LIMIT 1; + END IF; + END $$", + ) + .execute(&db) + .await?; + + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + // Keys for the same role: one pool scope when only the sslmode differs, two + // when the database does, which the role's limit still counts together. + let run = |dbname: &str, sslmode: &str| { + RunJob::from(JobPayload::Code(RawCode { + hash: None, + content: "SELECT 1 as n;".into(), + path: None, + lock: None, + language: ScriptLang::Postgresql, + cache_ttl: None, + cache_ignore_s3_path: None, + dedicated_worker: None, + concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default( + ) + .into(), + debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), + modules: None, + tag: None, + })) + .arg( + "database", + json!({"host": "localhost", "port": 5432, "dbname": dbname, + "user": "wm_pg_cache_one_conn", "password": "changeme", "sslmode": sslmode}), + ) + .run_until_complete(&db, false, port) + }; + + // The evicted connection's backend exits asynchronously, so a fresh + // connection can briefly still count it. Retrying absorbs that; without the + // eviction the cached connection stays open for 60s and every retry fails. + for (dbname, sslmode) in [ + ("windmill", "disable"), + ("windmill", "prefer"), + ("postgres", "disable"), + ("windmill", "disable"), + ] { + let mut result = json!(null); + for _ in 0..5 { + result = run(dbname, sslmode).await.json_result().unwrap(); + if result == json!([{"n": 1}]) { + break; + } + tokio::time::sleep(std::time::Duration::from_millis(200)).await; + } + assert_eq!(result, json!([{"n": 1}]), "{dbname} sslmode={sslmode}"); + } + + clear_pg_cache().await; + Ok(()) +} + +/// A transaction a script leaves open must not carry over into the next job +/// that would reuse the connection. +#[sqlx::test(fixtures("base"))] +#[serial(pg_cache)] +async fn test_postgresql_open_transaction_not_reused(db: Pool) -> anyhow::Result<()> { + use windmill_worker::pg_executor::clear_pg_cache; + + initialize_tracing().await; + clear_pg_cache().await; + + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let dbname: String = sqlx::query_scalar("SELECT current_database()") + .fetch_one(&db) + .await?; + + let run = |content: &str| { + RunJob::from(JobPayload::Code(RawCode { + hash: None, + content: content.to_string(), + path: None, + lock: None, + language: ScriptLang::Postgresql, + cache_ttl: None, + cache_ignore_s3_path: None, + dedicated_worker: None, + concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default( + ) + .into(), + debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), + modules: None, + tag: None, + })) + .arg( + "database", + json!({"host": "localhost", "port": 5432, "dbname": dbname, "user": "postgres", "password": "changeme"}), + ) + .run_until_complete(&db, false, port) + }; + + run("SELECT 1 as n;").await.json_result().unwrap(); + let opened = run("BEGIN; SELECT 1 as n;").await.json_result().unwrap(); + assert_eq!(opened, json!([{"n": 1}])); + run("SELECT 2 as n;").await.json_result().unwrap(); + + // A connection kept in the cache with the transaction open would sit idle + // in it, holding its locks, and run the next job inside it. + let mut idle_in_transaction = -1; + for _ in 0..20 { + idle_in_transaction = sqlx::query_scalar::<_, i64>( + "SELECT count(*) FROM pg_stat_activity + WHERE datname = current_database() AND state LIKE 'idle in transaction%'", + ) + .fetch_one(&db) + .await?; + if idle_in_transaction == 0 { + break; + } + tokio::time::sleep(std::time::Duration::from_millis(100)).await; + } + assert_eq!(idle_in_transaction, 0); + + clear_pg_cache().await; + Ok(()) +} + +/// Closing the connection of a cancelled job does not stop its query: the server +/// only notices once it next writes to the client. +#[sqlx::test(fixtures("base"))] +#[serial(pg_cache)] +async fn test_postgresql_cancelled_job_stops_its_query(db: Pool) -> anyhow::Result<()> { + use windmill_worker::pg_executor::clear_pg_cache; + + initialize_tracing().await; + clear_pg_cache().await; + + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let dbname: String = sqlx::query_scalar("SELECT current_database()") + .fetch_one(&db) + .await?; + let still_running = "SELECT count(*) FROM pg_stat_activity + WHERE datname = current_database() AND state = 'active' + AND pid <> pg_backend_pid() AND query LIKE '%pg_sleep(30)%'"; + + let cancelled = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)); + let job = RunJob::from(JobPayload::Code(RawCode { + hash: None, + content: "SELECT pg_sleep(30);".to_string(), + path: None, + lock: None, + language: ScriptLang::Postgresql, + cache_ttl: None, + cache_ignore_s3_path: None, + dedicated_worker: None, + concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default() + .into(), + debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), + modules: None, + tag: None, + })) + .arg( + "database", + json!({"host": "localhost", "port": 5432, "dbname": dbname, "user": "postgres", "password": "changeme"}), + ) + .run_until_complete_with(&db, false, port, |id| { + let (db, cancelled) = (db.clone(), cancelled.clone()); + async move { + tokio::spawn(async move { + while sqlx::query_scalar::<_, i64>(still_running) + .fetch_one(&db) + .await + .unwrap() + == 0 + { + tokio::time::sleep(std::time::Duration::from_millis(50)).await; + } + sqlx::query("UPDATE v2_job_queue SET canceled_by = 'test-user' WHERE id = $1") + .bind(id) + .execute(&db) + .await + .unwrap(); + cancelled.store(true, std::sync::atomic::Ordering::Relaxed); + }); + } + }) + .await; + assert!(!job.success); + // Otherwise the job failed before its query ran, and there was nothing to stop. + assert!(cancelled.load(std::sync::atomic::Ordering::Relaxed)); + + let mut running = -1; + for _ in 0..50 { + running = sqlx::query_scalar::<_, i64>(still_running) + .fetch_one(&db) + .await?; + if running == 0 { + break; + } + tokio::time::sleep(std::time::Duration::from_millis(100)).await; + } + assert_eq!(running, 0); + + clear_pg_cache().await; + Ok(()) +} + +/// A worker alternating between two databases keeps a connection for each. +#[sqlx::test(fixtures("base"))] +#[serial(pg_cache)] +async fn test_postgresql_cache_keeps_a_connection_per_database( + db: Pool, +) -> anyhow::Result<()> { + use std::sync::atomic::Ordering; + use windmill_worker::pg_executor::{clear_pg_cache, CACHE_HITS}; + + initialize_tracing().await; + clear_pg_cache().await; + + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + let run = |dbname: &str| { + RunJob::from(JobPayload::Code(RawCode { + hash: None, + content: "SELECT current_database() as d;".into(), + path: None, + lock: None, + language: ScriptLang::Postgresql, + cache_ttl: None, + cache_ignore_s3_path: None, + dedicated_worker: None, + concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default( + ) + .into(), + debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), + modules: None, + tag: None, + })) + .arg( + "database", + json!({"host": "localhost", "port": 5432, "dbname": dbname, + "user": "postgres", "password": "changeme"}), + ) + .run_until_complete(&db, false, port) + }; + + let hits_before = CACHE_HITS.load(Ordering::Relaxed); + for dbname in ["windmill", "postgres", "windmill", "postgres"] { + let result = run(dbname).await.json_result().unwrap(); + assert_eq!(result, json!([{"d": dbname}])); + } + let hits = CACHE_HITS.load(Ordering::Relaxed) - hits_before; + assert_eq!( + hits, 2, + "the last two jobs should reuse a cached connection" + ); + + clear_pg_cache().await; + Ok(()) +} + /// Runs multiple PG jobs through a SINGLE worker (like production) to verify /// that SET ROLE / search_path changes do not leak across jobs. #[sqlx::test(fixtures("base"))] @@ -5653,6 +5941,7 @@ async fn test_flow_tag_judged_as_written_before_preprocessor( apply_preprocessor: true, version: 1443253234253456, labels: None, + job_token_scopes: None, }) .as_user("test-user-2", "test2@windmill.dev") .run_until_complete(&db, false, port) diff --git a/backend/windmill-ai/src/ai_providers.rs b/backend/windmill-ai/src/ai_providers.rs index a4dcbc4f35..4cd41f34a8 100644 --- a/backend/windmill-ai/src/ai_providers.rs +++ b/backend/windmill-ai/src/ai_providers.rs @@ -39,6 +39,7 @@ lazy_static::lazy_static! { pub const OPENAI_BASE_URL: &str = "https://api.openai.com/v1"; pub const DEEPSEEK_BASE_URL: &str = "https://api.deepseek.com/v1"; pub const GOOGLE_AI_BASE_URL: &str = "https://generativelanguage.googleapis.com/v1beta"; +pub const TYPESAFE_BASE_URL: &str = "https://api.typesafe.ai/v1"; /// Hosts that serve the OpenAI API with Azure conventions: Azure OpenAI /// (`*.openai.azure.com`), AI Foundry (`*.services.ai.azure.com`, @@ -122,9 +123,35 @@ pub enum AIProvider { CustomAI, #[serde(rename = "aws_bedrock")] AWSBedrock, + /// TypeSafe's Jev decision model. It answers typed questions rather than chatting, so only + /// an AI decision runs it (`run_systemone`), never the agent loop. + TypeSafe, + /// Cloudflare Workers AI, for its Jev-compatible decision models (Clef). Decisions only, as + /// for TypeSafe; its base URL comes from the resource's account id. + Cloudflare, } +/// The Workers AI models URL of a Cloudflare account. The id goes into the path, so it is held to +/// the characters an account id has rather than trusted to stay inside its segment. +pub fn cloudflare_workers_ai_base_url(account_id: Option<&str>) -> Result { + match account_id { + Some(id) if !id.is_empty() && id.chars().all(|c| c.is_ascii_alphanumeric()) => Ok(format!( + "https://api.cloudflare.com/client/v4/accounts/{id}/ai/run/@cf/cloudflare" + )), + Some(_) => Err(Error::BadRequest( + "The Cloudflare account_id must be letters and digits only".to_string(), + )), + None => Err(Error::BadRequest( + "A Cloudflare resource needs an account_id".to_string(), + )), + } +} impl AIProvider { + /// A provider that answers decisions (typed questions) and serves no chat. + pub fn is_decision_provider(&self) -> bool { + matches!(self, AIProvider::TypeSafe | AIProvider::Cloudflare) + } + /// Get the base URL for the AI provider pub async fn get_base_url(&self, resource_base_url: Option, db: &DB) -> Result { if let Some(base_url) = resource_base_url { @@ -177,6 +204,10 @@ impl AIProvider { AIProvider::TogetherAI => Ok("https://api.together.xyz/v1".to_string()), AIProvider::Anthropic => Ok("https://api.anthropic.com/v1".to_string()), AIProvider::Mistral => Ok("https://api.mistral.ai/v1".to_string()), + AIProvider::TypeSafe => Ok(TYPESAFE_BASE_URL.to_string()), + AIProvider::Cloudflare => Err(Error::BadRequest( + "A Cloudflare resource needs an account_id".to_string(), + )), p @ (AIProvider::CustomAI | AIProvider::AzureOpenAI | AIProvider::AzureFoundry) => { Err(Error::BadRequest(format!( "{:?} provider requires a base URL in the resource", diff --git a/backend/windmill-ai/src/decision.rs b/backend/windmill-ai/src/decision.rs new file mode 100644 index 0000000000..31b525cf2d --- /dev/null +++ b/backend/windmill-ai/src/decision.rs @@ -0,0 +1,258 @@ +//! AI decision steps: one call to a System One endpoint (TypeSafe's Jev, or Cloudflare's +//! Jev-compatible Clef on Workers AI), which answers typed questions (`choice`, `score`, `noul`) +//! about a state with calibrated probabilities instead of text. + +use std::time::Duration; + +use serde::{Deserialize, Serialize}; +use serde_json::{value::RawValue, Map, Value}; +use windmill_common::error::{Error, Result}; + +use crate::{ + ai_providers::AIProvider, + credentials::ProviderCredentials, + proxy::{common_outbound_headers, retain_effective_credentials}, + types::{ProviderWithResource, TokenUsage}, + utils::pinned_ai_client_for, +}; + +/// The inputs of an AI decision step or tool, as its `input_transforms` resolve them. +#[derive(Deserialize, Debug)] +pub struct AIDecisionArgs { + pub provider: ProviderWithResource, + #[serde(default)] + pub state: Option, + #[serde(default)] + pub questions: Option, +} + +/// The alias TypeSafe keeps on its current model, sent when the step names none. +pub const TYPESAFE_DEFAULT_MODEL: &str = "jev-latest"; +/// Cloudflare's larger decision model, run when the step names none. +pub const CLOUDFLARE_DEFAULT_MODEL: &str = "clef"; + +#[derive(Serialize)] +struct SystemOneRequest<'a> { + model: &'a str, + state: &'a Value, + questions: &'a Map, +} + +#[derive(Deserialize)] +struct SystemOneResponse { + #[serde(default)] + model: Option, + answers: Box, + #[serde(default)] + usage: Option, +} + +/// Workers AI wraps the System One response in its API envelope. +#[derive(Deserialize)] +struct CloudflareEnvelope { + result: SystemOneResponse, +} + +#[derive(Deserialize)] +struct SystemOneUsage { + input_tokens: Option, + output_tokens: Option, +} + +/// A decision's result. `output` holds the answers keyed by question name, where an agent's +/// holds its answer, so a later step reads both as `results..output`. +#[derive(Serialize)] +pub struct DecisionResult { + pub output: Box, + /// The version that answered, which `jev-latest` resolves to. + #[serde(skip_serializing_if = "Option::is_none")] + pub model: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub usage: Option, +} + +/// The state and questions of a decision, refused here rather than by the endpoint when +/// either is missing or of a shape TypeSafe rejects: a flow that passes an empty expression result +/// should read what it forgot, without a request going out. +pub fn decision_inputs<'a>( + state: Option<&'a Value>, + questions: Option<&'a Value>, +) -> Result<(&'a Value, &'a Map)> { + let state = match state { + None | Some(Value::Null) => { + return Err(Error::BadRequest( + "'state' must be provided for an AI decision".to_string(), + )) + } + Some(Value::String(s)) if s.trim().is_empty() => { + return Err(Error::BadRequest( + "'state' must be provided for an AI decision".to_string(), + )) + } + Some(state @ (Value::String(_) | Value::Object(_) | Value::Array(_))) => state, + Some(_) => { + return Err(Error::BadRequest( + "'state' must be a text, an object or an array for an AI decision".to_string(), + )) + } + }; + let questions = match questions { + Some(Value::Object(questions)) if !questions.is_empty() => questions, + _ => { + return Err(Error::BadRequest( + "'questions' must be an object naming at least one question for an AI decision" + .to_string(), + )) + } + }; + Ok((state, questions)) +} + +pub async fn run_systemone( + credentials: &ProviderCredentials, + model: &str, + state: &Value, + questions: &Map, + timeout: Duration, +) -> Result { + let cloudflare = credentials.provider == AIProvider::Cloudflare; + let model = match model.trim() { + "" if cloudflare => CLOUDFLARE_DEFAULT_MODEL, + "" => TYPESAFE_DEFAULT_MODEL, + model => model, + }; + let base = credentials.base_url.trim_end_matches('/'); + // Workers AI serves each model at its own URL; the model goes into the path, so it is held to + // the characters a model id has. + let endpoint = if cloudflare { + if !model + .chars() + .all(|c| c.is_ascii_alphanumeric() || matches!(c, '-' | '_' | '.')) + { + return Err(Error::BadRequest(format!( + "Invalid Cloudflare decision model: {model}" + ))); + } + format!("{base}/{model}") + } else { + format!("{base}/systemone") + }; + let vendor = if cloudflare { "Cloudflare" } else { "TypeSafe" }; + let auth_headers = retain_effective_credentials( + credentials, + vec![( + "Authorization", + format!( + "Bearer {}", + credentials.api_key.as_deref().unwrap_or_default() + ), + )], + ); + + // The endpoint derives from the resource's base URL, so the connect is pinned to the + // address the SSRF check validated. + let client = pinned_ai_client_for(&credentials.base_url).await?; + let mut request = client.post(&endpoint).timeout(timeout); + for (name, value) in auth_headers { + request = request.header(name, value); + } + for (name, value) in common_outbound_headers(credentials) { + request = request.header(name, value); + } + + let response = request + .json(&SystemOneRequest { model, state, questions }) + .send() + .await + .map_err(|e| Error::ExecutionErr(format!("Failed to call {vendor}: {e}")))?; + + let status = response.status(); + if !status.is_success() { + let body = response + .text() + .await + .unwrap_or_else(|_| "".to_string()); + return Err(Error::ExecutionErr(format!( + "{vendor} error calling {endpoint}: {status} - {body}" + ))); + } + + let body = response + .text() + .await + .map_err(|e| Error::ExecutionErr(format!("Failed to read the {vendor} response: {e}")))?; + // Tried in turn rather than as an untagged enum, which cannot hold the raw `answers`. + let parsed = match serde_json::from_str::(&body) { + Ok(parsed) => parsed, + Err(e) => match serde_json::from_str::(&body) { + Ok(envelope) => envelope.result, + Err(_) => { + return Err(Error::ExecutionErr(format!( + "Unexpected {vendor} response ({e}): {body}" + ))) + } + }, + }; + + Ok(DecisionResult { + output: parsed.answers, + model: parsed.model, + // Dropped when empty, as an agent's is, so `usage` is either counted or absent. + usage: parsed + .usage + .map(|u| TokenUsage::from_input_output(u.input_tokens, u.output_tokens)) + .filter(|u| !u.is_empty()), + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + /// The account id goes into the URL path, so anything that could leave its segment is refused. + #[test] + fn cloudflare_account_id_stays_in_its_segment() { + use crate::ai_providers::cloudflare_workers_ai_base_url; + assert_eq!( + cloudflare_workers_ai_base_url(Some("abc123")).unwrap(), + "https://api.cloudflare.com/client/v4/accounts/abc123/ai/run/@cf/cloudflare" + ); + for id in [None, Some("a/b"), Some("../x"), Some("a?b"), Some("")] { + assert!(cloudflare_workers_ai_base_url(id).is_err(), "{id:?}"); + } + } + + /// Refused are the values a flow produces when it forgot the input (no key, `null`, a blank + /// string from an empty expression) and the scalars TypeSafe rejects. A text, an object or an + /// array is a state, whatever it holds. + #[test] + fn decision_inputs_refuse_what_typesafe_would() { + let questions = json!({"urgent": {"type": "noul", "instructions": "Is it urgent?"}}); + for state in [ + None, + Some(json!(null)), + Some(json!(" ")), + Some(json!(false)), + Some(json!(0)), + ] { + let err = decision_inputs(state.as_ref(), Some(&questions)).unwrap_err(); + assert!(err.to_string().contains("'state'"), "{err}"); + } + for state in [json!("0"), json!({}), json!([]), json!({"m": "hi"})] { + assert!( + decision_inputs(Some(&state), Some(&questions)).is_ok(), + "{state}" + ); + } + for questions in [ + None, + Some(json!({})), + Some(json!([])), + Some(json!("urgent")), + ] { + let err = decision_inputs(Some(&json!("hi")), questions.as_ref()).unwrap_err(); + assert!(err.to_string().contains("'questions'"), "{err}"); + } + } +} diff --git a/backend/windmill-ai/src/image_handler.rs b/backend/windmill-ai/src/image_handler.rs index 5bd9794da5..a4f360cdb4 100644 --- a/backend/windmill-ai/src/image_handler.rs +++ b/backend/windmill-ai/src/image_handler.rs @@ -47,6 +47,9 @@ pub async fn upload_image_to_s3( }) } +// Bounds each attachment's download before base64 expansion; provider limits may be lower. +const MAX_S3_MEDIA_BYTES: usize = 30 * 1024 * 1024; + /// Download an S3 image and convert it to a base64 data URL. /// /// The caller must provide an AuthedClient authorized for `workspace_id`. @@ -57,7 +60,12 @@ pub async fn download_and_encode_s3_image( ) -> Result<(String, String), Error> { // Download the image from S3 let image_bytes = client - .download_s3_file(workspace_id, &image.s3, image.storage.clone()) + .download_s3_file( + workspace_id, + &image.s3, + image.storage.clone(), + Some(MAX_S3_MEDIA_BYTES), + ) .await .map_err(|e| Error::internal_err(format!("Failed to download S3 image: {}", e)))?; diff --git a/backend/windmill-ai/src/lib.rs b/backend/windmill-ai/src/lib.rs index e81d030ed3..ccc40da432 100644 --- a/backend/windmill-ai/src/lib.rs +++ b/backend/windmill-ai/src/lib.rs @@ -5,6 +5,7 @@ pub mod ai_google; pub mod ai_providers; pub mod ai_types; pub mod credentials; +pub mod decision; pub mod image_handler; pub mod model_context; pub mod providers; diff --git a/backend/windmill-ai/src/providers/mod.rs b/backend/windmill-ai/src/providers/mod.rs index b5b6281264..e08b038bb0 100644 --- a/backend/windmill-ai/src/providers/mod.rs +++ b/backend/windmill-ai/src/providers/mod.rs @@ -204,6 +204,12 @@ pub fn create_query_builder( AIProvider::AzureFoundry if AIProvider::is_anthropic_model(model) => Box::new( AnthropicQueryBuilder::new(AIProvider::AzureFoundry, AIPlatform::Standard), ), + // Only for forwarding through the API proxy, which is bearer auth on `/` as + // TypeSafe and Cloudflare expect. Neither serves a chat route for its decision models, and + // `run_agent` refuses both before a chat request could be built. + AIProvider::TypeSafe | AIProvider::Cloudflare => { + Box::new(OtherQueryBuilder::new(credentials.provider.clone())) + } _ => Box::new(OtherQueryBuilder::new(credentials.provider.clone())), } } diff --git a/backend/windmill-ai/src/proxy.rs b/backend/windmill-ai/src/proxy.rs index 3af8f88a1d..887cab0f3e 100644 --- a/backend/windmill-ai/src/proxy.rs +++ b/backend/windmill-ai/src/proxy.rs @@ -72,7 +72,9 @@ pub fn proxy_execution_mode(provider: &AIProvider) -> ProxyExecutionMode { | AIProvider::Groq | AIProvider::OpenRouter | AIProvider::TogetherAI - | AIProvider::CustomAI => ProxyExecutionMode::HttpForward, + | AIProvider::CustomAI + | AIProvider::TypeSafe + | AIProvider::Cloudflare => ProxyExecutionMode::HttpForward, AIProvider::GoogleAI => ProxyExecutionMode::NativeGoogleAi, AIProvider::AWSBedrock => ProxyExecutionMode::NativeAwsBedrock, } @@ -444,6 +446,8 @@ mod tests { (AIProvider::OpenRouter, ProxyExecutionMode::HttpForward), (AIProvider::TogetherAI, ProxyExecutionMode::HttpForward), (AIProvider::CustomAI, ProxyExecutionMode::HttpForward), + (AIProvider::TypeSafe, ProxyExecutionMode::HttpForward), + (AIProvider::Cloudflare, ProxyExecutionMode::HttpForward), (AIProvider::GoogleAI, ProxyExecutionMode::NativeGoogleAi), (AIProvider::AWSBedrock, ProxyExecutionMode::NativeAwsBedrock), ]; diff --git a/backend/windmill-ai/src/query_builder.rs b/backend/windmill-ai/src/query_builder.rs index 986c433f42..a08d01dadd 100644 --- a/backend/windmill-ai/src/query_builder.rs +++ b/backend/windmill-ai/src/query_builder.rs @@ -1,5 +1,7 @@ +use std::borrow::Cow; + use async_trait::async_trait; -use windmill_common::{client::AuthedClient, error::Error}; +use windmill_common::{client::AuthedClient, error::Error, utils::strip_json_nul}; use windmill_types::s3::S3Object; use crate::ai_types::OpenAIToolCall; @@ -92,6 +94,39 @@ pub enum ParsedResponse { }, } +impl ParsedResponse { + /// Drops every U+0000 the model emitted. Postgres stores neither a NUL in `text` nor its + /// `\u0000` escape in `jsonb`, and this output is written to both: tool job args, the flow + /// status, agent memory, conversation rows. + pub fn without_nul(mut self) -> Self { + fn strip(s: &mut String) { + if s.contains('\0') { + s.retain(|c| c != '\0'); + } + } + if let ParsedResponse::Text { content, reasoning, tool_calls, annotations, .. } = &mut self + { + for text in [content, reasoning].into_iter().flatten() { + strip(text); + } + for annotation in annotations { + strip(&mut annotation.url); + annotation.title.as_mut().into_iter().for_each(strip); + } + for tool_call in tool_calls { + strip(&mut tool_call.id); + strip(&mut tool_call.function.name); + // Arguments are serialized JSON, where a NUL is spelled as its escape. + strip(&mut tool_call.function.arguments); + if let Cow::Owned(stripped) = strip_json_nul(&tool_call.function.arguments) { + tool_call.function.arguments = stripped; + } + } + } + self + } +} + /// Trait for streaming AI events to a sink (e.g., database persistence). /// Implemented by the worker's StreamEventProcessor. #[async_trait] diff --git a/backend/windmill-ai/src/types.rs b/backend/windmill-ai/src/types.rs index 7e2de09e01..e591e85159 100644 --- a/backend/windmill-ai/src/types.rs +++ b/backend/windmill-ai/src/types.rs @@ -269,6 +269,12 @@ pub struct ProviderResource { /// credentials exchange this crate does not perform. #[serde(default, deserialize_with = "empty_string_as_none")] pub token_url: Option, + /// The API token of a `cloudflare` resource, which names it `token` rather than `api_key`. + #[serde(default, deserialize_with = "empty_string_as_none")] + pub token: Option, + /// The account of a `cloudflare` resource, which its Workers AI URL is built from. + #[serde(default, deserialize_with = "empty_string_as_none")] + pub account_id: Option, } #[derive(Deserialize, Debug)] @@ -301,6 +307,11 @@ impl ProviderWithResource { } pub async fn get_base_url(&self, db: &DB) -> Result { + if self.kind == AIProvider::Cloudflare && self.resource.base_url.is_none() { + return crate::ai_providers::cloudflare_workers_ai_base_url( + self.resource.account_id.as_deref(), + ); + } self.kind .get_base_url(self.resource.base_url.clone(), db) .await @@ -320,7 +331,14 @@ impl ProviderWithResource { Ok(ProviderCredentials { provider: self.kind.clone(), base_url, - api_key: self.resource.api_key.clone(), + api_key: match self.kind { + AIProvider::Cloudflare => self + .resource + .api_key + .clone() + .or_else(|| self.resource.token.clone()), + _ => self.resource.api_key.clone(), + }, access_token: None, organization_id: None, user: None, diff --git a/backend/windmill-api-auth/src/auth.rs b/backend/windmill-api-auth/src/auth.rs index 869f40e14c..720111e89d 100644 --- a/backend/windmill-api-auth/src/auth.rs +++ b/backend/windmill-api-auth/src/auth.rs @@ -104,6 +104,32 @@ pub struct ExpiringAuthCache { pub job_id: Option, } +/// Whether `target` belongs to the flow run of the job `job_id`: the job itself, one of its +/// ancestors, or a job whose parent or root is one of them (a sibling step, a loop or branch +/// iteration, any job pushed as a child of one of them). A lookup that fails reads as no. +pub async fn job_in_same_flow_run(db: &DB, job_id: uuid::Uuid, target: uuid::Uuid) -> bool { + if job_id == target { + return true; + } + sqlx::query_scalar!( + "SELECT EXISTS ( + SELECT 1 FROM v2_job o, v2_job t, + LATERAL (SELECT ARRAY_REMOVE(ARRAY[o.id, o.parent_job, o.root_job, + o.flow_innermost_root_job], NULL) AS run) l + WHERE o.id = $1 AND t.id = $2 AND t.workspace_id = o.workspace_id + AND (t.id = ANY(l.run) OR t.parent_job = ANY(l.run) + OR t.root_job = ANY(l.run) OR t.flow_innermost_root_job = ANY(l.run)) + )", + job_id, + target + ) + .fetch_one(db) + .await + .ok() + .flatten() + .unwrap_or(false) +} + pub struct AuthCache { db: DB, superadmin_secret: Option, @@ -1131,8 +1157,20 @@ pub async fn resolve_opt_job_authed( return Err((err, parts)); } } + let own_job_runtime_route = match opt_job_authed.job_id { + Some(job_id) if opt_job_authed.authed.scopes.is_some() => { + crate::scopes::is_own_job_runtime_route(path, method, job_id) + || match crate::scopes::flow_run_read_route_job(path, method) { + Some(target) => { + job_in_same_flow_run(&cache.db, job_id, target).await + } + None => false, + } + } + _ => false, + }; let authed = &mut opt_job_authed.authed; - if authed.scopes.is_some() { + if authed.scopes.is_some() && !own_job_runtime_route { transform_old_scope_to_new_scope(authed.scopes.as_mut()); if let Err(err) = crate::scopes::check_scopes_for_route( diff --git a/backend/windmill-api-auth/src/lib.rs b/backend/windmill-api-auth/src/lib.rs index 12ad5f2654..30c3bb65af 100644 --- a/backend/windmill-api-auth/src/lib.rs +++ b/backend/windmill-api-auth/src/lib.rs @@ -35,6 +35,7 @@ use windmill_common::{ }; use scopes::ScopeDefinition; +use windmill_common::scopes::scope_contains; // Re-export key auth types and functions pub use auth::{ @@ -475,11 +476,11 @@ pub fn is_effectively_unscoped(scopes: Option<&[String]>) -> bool { /// refused. An admin check is not a substitute — it answers for the user behind the /// token, not for the token's own scopes. /// -/// This bounds the token making the request, not every route to the key. A job token -/// is minted unscoped from its owner's privileges, so a `jobs:run` token still reaches -/// the key indirectly by running a job as a workspace admin — the same property that -/// lets git-sync export it. Confining that means not inheriting unscoped privilege -/// into job tokens, which is a far wider change than this guard. +/// This bounds the token making the request, not every route to the key. The jobs a +/// scoped user token runs get a token with their owner's privileges, capped only by the +/// runnable's own `job_token_scopes` (see [`caller_scope_ceiling`]), so a `jobs:run` +/// token still reaches the key indirectly by running an unrestricted job as a workspace +/// admin — the same property that lets git-sync export it. pub fn forbid_scoped_token_workspace_key(authed: &ApiAuthed) -> error::Result<()> { if is_effectively_unscoped(authed.scopes.as_deref()) { return Ok(()); @@ -492,6 +493,50 @@ pub fn forbid_scoped_token_workspace_key(authed: &ApiAuthed) -> error::Result<() )) } +/// For a route that authenticates its token itself rather than through the route layer +/// (which checks scopes): a job token restricted by `job_token_scopes` must still hold +/// `required` there. Other credentials keep the access those routes always gave them. +pub fn check_job_token_scope(authed: &ApiAuthed, required: F) -> error::Result<()> +where + F: FnOnce() -> String, +{ + if authed.job_id.is_some() { + check_scopes(authed, required) + } else { + Ok(()) + } +} + +/// The cap on the token of a job pushed on `authed`'s request, to pass to `push` as its +/// scope ceiling. A job token caps every job it starts at its own job's scopes, so a +/// restricted job cannot widen its reach by running another one; the job row is the +/// source rather than the token's claims, which the MCP proxy narrows per route. Any +/// other credential caps nothing: the jobs it runs act as their owner, as they always did. +pub async fn caller_scope_ceiling( + db: &windmill_common::DB, + authed: &ApiAuthed, +) -> error::Result>> { + let Some(job_id) = authed.job_id else { + return Ok(None); + }; + // A job token is minted with scopes exactly when its job is restricted, so an unscoped one + // has nothing to read. + if authed.scopes.is_none() { + return Ok(None); + } + let row = sqlx::query_scalar!( + "SELECT job_token_scopes FROM job_perms WHERE job_id = $1", + job_id + ) + .fetch_optional(db) + .await?; + Ok(match row { + Some(scopes) => scopes, + // The job left the queue and its row was swept: only the token's own claims are left. + None => authed.scopes.clone(), + }) +} + /// Enforce monotonic privilege when a token lifecycle endpoint mints or rescopes /// a credential on behalf of `authed`: the resulting credential must never be /// more privileged than the caller's own token. @@ -604,83 +649,6 @@ fn first_filter_tags(scopes: Option<&[String]>) -> Option> { }) } -/// Whether `caller` grants at least everything `requested` grants (directional -/// containment). -/// -/// This is intentionally NOT `ScopeDefinition::includes`: that method answers -/// "does this scope grant access to a required action" using OR semantics over -/// resources (any overlap counts, and a `*` on either side matches), which is -/// correct for access checks but unsafe for subset checks — it would let a -/// token scoped to `scripts:read:f/team/a` mint `scripts:read:*` or -/// `scripts:read:f/team/a,f/other/b`. Subset containment instead requires that -/// EVERY requested resource is covered by SOME caller resource. -fn scope_contains(caller: &ScopeDefinition, requested: &ScopeDefinition) -> bool { - if caller.domain != requested.domain { - return false; - } - - // write subsumes read; otherwise the action must match exactly. - match (caller.action.as_str(), requested.action.as_str()) { - (c, r) if c == r || (c == "write" && r == "read") => {} - // Apps only: `write` covers `run` (see `ScopeDefinition::includes`), so an - // app-editor token can mint the narrower run-only credential. - ("write", "run") if caller.domain == "apps" => {} - _ => return false, - } - - if caller.domain == "jobs" && caller.action == "run" { - match (&caller.kind, &requested.kind) { - (Some(caller_kind), Some(requested_kind)) if caller_kind != requested_kind => { - return false - } - // Caller pinned to a kind, but the request covers any kind. - (Some(_), None) => return false, - _ => {} - } - } - - match (&caller.resource, &requested.resource) { - // Caller is unrestricted on resources: covers everything. - (None, _) => true, - // Caller is resource-restricted but the request is not: broader, unless the - // caller lists `*` and so already spans every path. Kept in step with - // `ScopeDefinition::includes`, which accepts that same grant for a - // whole-collection read: what a token may exercise, it may also delegate. - (Some(caller_resources), None) => caller_resources.iter().any(|r| r == "*"), - (Some(caller_resources), Some(requested_resources)) => { - resource_set_contains(caller_resources, requested_resources) - } - } -} - -/// Every resource in `requested` must be covered by some resource in `caller`. -fn resource_set_contains(caller: &[String], requested: &[String]) -> bool { - if caller.iter().any(|r| r == "*") { - return true; - } - requested - .iter() - .all(|req| req != "*" && caller.iter().any(|c| resource_covers(c, req))) -} - -/// Directional: does the single caller resource pattern cover `requested`? -/// `caller` may be an exact path or a `/*` subtree wildcard; `requested` -/// may itself be a subtree wildcard, in which case the whole requested subtree -/// must fall within the caller's subtree. -fn resource_covers(caller: &str, requested: &str) -> bool { - if caller == requested { - return true; - } - let Some(prefix) = caller.strip_suffix("/*") else { - // An exact caller resource only covers itself (handled above). - return false; - }; - let requested_base = requested.strip_suffix("/*").unwrap_or(requested); - requested_base == prefix - || (requested_base.starts_with(prefix) - && requested_base.as_bytes().get(prefix.len()) == Some(&b'/')) -} - /// Returns a predicate that checks whether `path` is within the token's /// scope for `{domain}:{action}:{path}`. For tokens without scope /// restrictions (no scopes at all, or only `if_jobs:filter_tags:*` scopes), diff --git a/backend/windmill-api-auth/src/scopes.rs b/backend/windmill-api-auth/src/scopes.rs index e100cf88c6..dce6a0ff72 100644 --- a/backend/windmill-api-auth/src/scopes.rs +++ b/backend/windmill-api-auth/src/scopes.rs @@ -6,464 +6,9 @@ * LICENSE-AGPL for a copy of the license. */ -use itertools::Itertools; -use serde::{Deserialize, Serialize}; -use std::collections::HashSet; use windmill_common::error::{Error, Result}; -/// Comprehensive scope system for JWT token authorization -/// -/// Scopes follow the format: {domain}:{action}[:{resource}] -/// Examples: -/// - "jobs:read" - Read access to jobs -/// - "scripts:write:f/folder/*" - Write access to scripts in a folder -/// - "*" - Full access (superuser) - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct ScopeDefinition { - pub domain: String, - pub action: String, - pub kind: Option, // For jobs:run:kind (optional) - pub resource: Option>, -} - -impl ScopeDefinition { - pub fn new( - domain: &str, - action: &str, - kind: Option<&str>, - resource: Option>, - ) -> Self { - Self { - domain: domain.to_string(), - action: action.to_string(), - kind: kind.map(|s| s.to_string()), - resource: resource, - } - } - - pub fn from_scope_string(scope: &str) -> Result { - let parts: Vec<&str> = scope.split(':').collect(); - - let into_owned_vec = |resources: &str| -> Vec { - let resources = resources - .split(",") - .collect_vec() - .into_iter() - .map(ToOwned::to_owned) - .collect_vec(); - - resources - }; - - match parts.len() { - 2 => Ok(Self::new(parts[0], parts[1], None, None)), // domain:action - 3 => { - if parts[0] == "jobs" && parts[1] == "run" { - Ok(Self::new(parts[0], parts[1], Some(parts[2]), None)) - } else { - Ok(Self::new( - parts[0], - parts[1], - None, - Some(into_owned_vec(parts[2])), - )) - } - } - 4 => { - if parts[0] == "jobs" && parts[1] == "run" { - Ok(Self::new( - parts[0], - parts[1], - Some(parts[2]), - Some(into_owned_vec(parts[3])), - )) - } else { - Err(Error::BadRequest(format!( - "Invalid 4-part scope: {}", - scope - ))) - } - } - _ => Err(Error::BadRequest(format!( - "Invalid scope format: {}", - scope - ))), - } - } - - pub fn as_string(&self) -> String { - match (&self.kind, &self.resource) { - (Some(kind), Some(resource)) => { - format!( - "{}:{}:{}:{}", - self.domain, - self.action, - kind, - resource.join(",") - ) - } - (Some(kind), None) => { - format!("{}:{}:{}", self.domain, self.action, kind) - } - (None, Some(resource)) => { - format!("{}:{}:{}", self.domain, self.action, resource.join(",")) - } - (None, None) => format!("{}:{}", self.domain, self.action), - } - } - - pub fn includes(&self, other: &ScopeDefinition) -> bool { - if self.domain != other.domain { - return false; - } - - match (self.action.as_str(), other.action.as_str()) { - (a, b) if (a == "write" && b == "read") || (a == b) => {} - // Apps only: `write` can rewrite the app and its policy, so it also covers - // running its components. Not general — `jobs:write` must not grant - // `jobs:run`. The resource check below still confines it to the same app. - ("write", "run") if self.domain == "apps" => {} - _ => return false, - } - - if self.domain == "jobs" && self.action == "run" { - match (&self.kind, &other.kind) { - (Some(self_kind), Some(other_kind)) => { - if self_kind != other_kind { - return false; - } - } - (Some(_), None) => { - return false; - } - (None, _) => { - return true; - } - } - } - - match (&self.resource, &other.resource) { - (Some(self_resources), Some(other_resources)) => { - resources_match(self_resources, other_resources) - } - // A requirement naming no path is the whole domain, so only a grant that - // itself spans every path satisfies it. `*` is that grant — the scope UI - // accepts it as a resource path and `resources_match` already reads it as - // everything — while any listed path leaves the collection unauthorized. - (Some(self_resources), None) => self_resources.iter().any(|r| r == "*"), - (None, _) => true, - } - } -} - -fn resources_match(scope_resources: &[String], accepted_resources: &[String]) -> bool { - if scope_resources.contains(&"*".to_string()) || accepted_resources.contains(&"*".to_string()) { - return true; - } - - if scope_resources.len() <= 4 && accepted_resources.len() <= 4 { - return resources_match_small(scope_resources, accepted_resources); - } - - resources_match_large(scope_resources, accepted_resources) -} - -fn resources_match_small(scope_resources: &[String], accepted_resources: &[String]) -> bool { - for required in accepted_resources { - for scope_resource in scope_resources { - if resource_matches_pattern(scope_resource, required) { - return true; - } - } - } - false -} - -fn resources_match_large(scope_resources: &[String], accepted_resources: &[String]) -> bool { - let mut exact_matches = HashSet::new(); - let mut patterns = Vec::new(); - - for scope_resource in scope_resources { - if scope_resource.contains('*') { - patterns.push(scope_resource); - } else { - exact_matches.insert(scope_resource); - } - } - - for accepted_resource in accepted_resources { - if exact_matches.contains(accepted_resource) { - return true; - } - - for pattern in &patterns { - if resource_matches_pattern(pattern, accepted_resource) { - return true; - } - } - } - - false -} - -fn resource_matches_pattern(scope_resource: &str, accepted_resource: &str) -> bool { - if scope_resource == accepted_resource { - return true; - } - - let matches_wildcard = |pattern: &str, resource: &str| -> bool { - if !pattern.ends_with("/*") { - return false; - } - - let prefix = &pattern[..pattern.len() - 2]; - - if !resource.starts_with(prefix) { - return false; - } - - // If the resource is exactly the prefix, it matches - if resource.len() == prefix.len() { - return true; - } - - // If the resource is longer, the next character must be '/' for a valid match - // This prevents "u/user" from matching "u/use/*" - resource.chars().nth(prefix.len()) == Some('/') - }; - - // Check if either resource is a wildcard pattern and matches the other - matches_wildcard(scope_resource, accepted_resource) - || matches_wildcard(accepted_resource, scope_resource) -} - -// ───────────────────────────────────────────────────────────────── -// Route-level scope checking -// ───────────────────────────────────────────────────────────────── - -/// Available scope domains (top-level API categories) -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -pub enum ScopeDomain { - // Core resource domains - Jobs, - Scripts, - /// The `/data_metrics` catalog. Its own domain, NOT an alias of `Scripts`: a - /// `data_metrics:read` token must reach only this route, never the broader - /// `/scripts` routes (some of which do no further scope check). - DataMetrics, - Flows, - FlowConversations, - Apps, - Variables, - Resources, - Schedules, - Folders, - Users, - Groups, - Workspaces, - - // Trigger domains - HttpTriggers, - WebsocketTriggers, - KafkaTriggers, - NatsTriggers, - MqttTriggers, - AmqpTriggers, - SqsTriggers, - GcpTriggers, - AzureTriggers, - PostgresTriggers, - EmailTriggers, - - // Native trigger domains - NativeTriggers, - TriggersHistory, - - // System domains - Audit, - Settings, - Workers, - ServiceLogs, - Configs, - OAuth, - AI, - AiEvals, // AI agent eval datasets - - Indexer, - Teams, // Microsoft Teams integration - GitSync, // Git synchronization - - // Special domains - Capture, // Webhook capture - Drafts, // Draft resources - Favorites, // User favorites - Inputs, // Input templates - JobHelpers, // Job helper functions - ConcurrencyGroups, // Concurrency groups - Oidc, // OpenID Connect - Openapi, // OpenAPI generation - - // Additional domains - Acls, // Granular access control lists - RawApps, // Raw application data - AgentWorkers, // Agent workers management - Mcp, // MCP - Docs, // Self-hosted documentation search (read-only) -} - -impl ScopeDomain { - pub fn as_str(&self) -> &'static str { - match self { - Self::Jobs => "jobs", - Self::Scripts => "scripts", - Self::DataMetrics => "data_metrics", - Self::Flows => "flows", - Self::FlowConversations => "flow_conversations", - Self::Apps => "apps", - Self::Variables => "variables", - Self::Resources => "resources", - Self::Schedules => "schedules", - Self::Folders => "folders", - Self::Users => "users", - Self::Groups => "groups", - Self::Workspaces => "workspaces", - Self::HttpTriggers => "http_triggers", - Self::WebsocketTriggers => "websocket_triggers", - Self::KafkaTriggers => "kafka_triggers", - Self::NatsTriggers => "nats_triggers", - Self::MqttTriggers => "mqtt_triggers", - Self::AmqpTriggers => "amqp_triggers", - Self::SqsTriggers => "sqs_triggers", - Self::GcpTriggers => "gcp_triggers", - Self::AzureTriggers => "azure_triggers", - Self::PostgresTriggers => "postgres_triggers", - Self::EmailTriggers => "email_triggers", - Self::NativeTriggers => "native_triggers", - Self::TriggersHistory => "triggers_history", - Self::Audit => "audit", - Self::Settings => "settings", - Self::Workers => "workers", - Self::ServiceLogs => "service_logs", - Self::Configs => "configs", - Self::OAuth => "oauth", - Self::AI => "ai", - Self::AiEvals => "ai_evals", - Self::Capture => "capture", - Self::Drafts => "drafts", - Self::Favorites => "favorites", - Self::Inputs => "inputs", - Self::JobHelpers => "job_helpers", - Self::ConcurrencyGroups => "concurrency_groups", - Self::Oidc => "oidc", - Self::Openapi => "openapi", - Self::Acls => "acls", - Self::RawApps => "raw_apps", - Self::AgentWorkers => "agent_workers", - Self::Indexer => "indexer", - Self::Teams => "teams", - Self::GitSync => "git_sync", - Self::Mcp => "mcp", - Self::Docs => "docs", - } - } - - pub fn from_str(s: &str) -> Option { - match s { - "jobs" | "jobs_u" => Some(Self::Jobs), - "scripts" => Some(Self::Scripts), - // A distinct domain, not an alias of `scripts` (see the enum variant): - // a `data_metrics:read` token must not reach the broader /scripts routes. - "data_metrics" => Some(Self::DataMetrics), - "flows" => Some(Self::Flows), - "flow_conversations" => Some(Self::FlowConversations), - "apps" | "apps_u" => Some(Self::Apps), - "variables" => Some(Self::Variables), - "resources" => Some(Self::Resources), - "schedules" => Some(Self::Schedules), - "folders" => Some(Self::Folders), - "users" => Some(Self::Users), - "groups" => Some(Self::Groups), - "workspaces" => Some(Self::Workspaces), - "http_triggers" => Some(Self::HttpTriggers), - "websocket_triggers" => Some(Self::WebsocketTriggers), - "kafka_triggers" => Some(Self::KafkaTriggers), - "nats_triggers" => Some(Self::NatsTriggers), - "mqtt_triggers" => Some(Self::MqttTriggers), - "amqp_triggers" => Some(Self::AmqpTriggers), - "sqs_triggers" => Some(Self::SqsTriggers), - "gcp_triggers" => Some(Self::GcpTriggers), - "azure_triggers" => Some(Self::AzureTriggers), - "postgres_triggers" => Some(Self::PostgresTriggers), - "email_triggers" => Some(Self::EmailTriggers), - "audit" => Some(Self::Audit), - "settings" => Some(Self::Settings), - "workers" => Some(Self::Workers), - "service_logs" => Some(Self::ServiceLogs), - "configs" => Some(Self::Configs), - "oauth" => Some(Self::OAuth), - "ai" => Some(Self::AI), - "ai_evals" => Some(Self::AiEvals), - "indexer" | "srch" => Some(Self::Indexer), - "teams" => Some(Self::Teams), - "native_triggers" => Some(Self::NativeTriggers), - "triggers_history" => Some(Self::TriggersHistory), - "git_sync" | "github_app" => Some(Self::GitSync), - "capture" => Some(Self::Capture), - "drafts" => Some(Self::Drafts), - "favorites" => Some(Self::Favorites), - "inputs" => Some(Self::Inputs), - "job_helpers" => Some(Self::JobHelpers), - "concurrency_groups" => Some(Self::ConcurrencyGroups), - "oidc" => Some(Self::Oidc), - "openapi" => Some(Self::Openapi), - "acls" => Some(Self::Acls), - "raw_apps" => Some(Self::RawApps), - "agent_workers" => Some(Self::AgentWorkers), - "mcp" => Some(Self::Mcp), - "docs" => Some(Self::Docs), - _ => None, - } - } -} - -/// Available scope actions -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -pub enum ScopeAction { - Read, // GET operations, list, view - Write, // POST, PUT, PATCH, DELETE operations, create, update, delete - Run, // Special action for running (scripts, flows, etc.) -} - -impl ScopeAction { - pub fn as_str(&self) -> &'static str { - match self { - Self::Read => "read", - Self::Write => "write", - Self::Run => "run", - } - } - - pub fn from_str(s: &str) -> Option { - match s { - "read" => Some(Self::Read), - "write" => Some(Self::Write), - "delete" => Some(Self::Write), - "run" => Some(Self::Run), - _ => None, - } - } - - /// Check if this action includes another action - /// Write includes Read - pub fn includes(&self, other: &ScopeAction) -> bool { - match (self, other) { - (ScopeAction::Write, ScopeAction::Read) => true, - (ScopeAction::Run, ScopeAction::Read) => true, - (a, b) => a == b, - } - } -} +pub use windmill_common::scopes::{ScopeAction, ScopeDefinition, ScopeDomain}; pub fn check_route_access( token_scopes: &[String], @@ -636,7 +181,38 @@ lazy_static::lazy_static! { }; } +/// The job routes a `jobs:cancel` scope reaches, as workspaced route suffixes. A +/// path-scoped `jobs:cancel:` is resource-blind here like every scope; the +/// handlers confine it through `job_cancel_path_confinement`, so a cancel route added +/// here without that check would serve a path-scoped token every job. +const CANCEL_PATH_ACTIONS: [&'static str; 4] = [ + "jobs_u/queue/cancel/", + "jobs_u/queue/force_cancel/", + "jobs_u/queue/cancel_persistent/", + "jobs/queue/cancel_selection", +]; + +fn is_cancel_route(method: &str, route_path: &str) -> bool { + if !method.eq_ignore_ascii_case("POST") { + return false; + } + let mut parts = route_path.splitn(5, '/'); + let (Some(""), Some("api"), Some("w"), Some(_), Some(suffix)) = ( + parts.next(), + parts.next(), + parts.next(), + parts.next(), + parts.next(), + ) else { + return false; + }; + CANCEL_PATH_ACTIONS.iter().any(|p| suffix.starts_with(p)) +} + fn map_http_method_to_action(method: &str, route_path: &str) -> ScopeAction { + if is_cancel_route(method, route_path) { + return ScopeAction::Cancel; + } if RUN_PATH_ACTIONS .iter() .any(|run_path| route_path.contains(run_path)) @@ -726,7 +302,7 @@ fn extract_domain_from_route( /// Every entry is keyed by a job id and confines an authenticated caller to its own /// runnable — through `require_job_read_access`, through its own `jobs:run:flows:` /// check, or, where an approval token or resume secret bypasses that gate, through a -/// direct `require_job_within_run_scope`. The one exception is +/// direct `require_job_within_read_scope`. The one exception is /// `jobs_u/get_root_job_id/`, which has no check at all but discloses only flow lineage, /// to anyone, authenticated or not. Workspace-wide enumeration (`jobs/list`, counts, /// exports) and credential minting (`job_view_token`) are deliberately absent — those are @@ -943,23 +519,163 @@ fn resource_metadata_route_allowed(suffix: &str) -> bool { || suffix.starts_with("resources/type/") } -/// The `jobs:run` scopes a token's job reads are confined to, or `None` when they are -/// not confined to particular runnables. +/// The by-id and list routes a path-scoped `jobs:read:` reaches, as workspaced +/// route suffixes. Every other read in the domain is refused to it: counts, exports, +/// uuid listings, queue positions and dispatch edges aggregate over the workspace, and +/// the view-token and resume-url routes mint a credential (`approval_info` withholds +/// its view token from a confined caller for the same reason). +/// +/// Each entry confines the caller in its handler, through `require_job_read_access` or +/// a direct `require_job_within_read_scope`. `jobs/result_by_id/` is absent because it +/// resolves a node up through the enclosing flows, past the job the check admitted. +/// A route added here without that check serves a path-scoped token every job. +const READ_PATH_SCOPED_BY_ID_PATHS: [&'static str; 25] = [ + "jobs_u/get/", + "jobs_u/get_logs/", + "jobs_u/get_completed_logs_tail/", + "jobs_u/get_flow_all_logs/", + "jobs_u/get_flow_all_logs_structured/", + "jobs_u/get_flow_all_results/", + "jobs_u/get_args/", + "jobs_u/get_flow_debug_info/", + "jobs_u/completed/get/", + "jobs_u/completed/get_result/", + "jobs_u/completed/get_result_maybe/", + "jobs_u/completed/get_timing/", + "jobs_u/dispatch_events/", + "jobs_u/getupdate/", + "jobs_u/getupdate_sse/", + "jobs_u/get_log_file/", + "jobs_u/get_flow/", + "jobs_u/flow/approval_info/", + "jobs/run_progress/", + "jobs/flow/user_states/", + "jobs/completed/get/", + "jobs/completed/get_result/", + "jobs/completed/get_result_maybe/", + "jobs/completed/get_timing/", + "jobs/get_otel_traces/", +]; + +/// Matched whole: `jobs/list` is also a prefix of the uuid listings. Their handlers +/// filter in SQL to [`job_list_path_filter`]. +const READ_PATH_SCOPED_LIST_PATHS: [&'static str; 3] = + ["jobs/list", "jobs/completed/list", "jobs/queue/list"]; + +fn is_read_path_scoped_route(suffix: &str) -> bool { + READ_PATH_SCOPED_LIST_PATHS.contains(&suffix) + || READ_PATH_SCOPED_BY_ID_PATHS + .iter() + .any(|p| suffix.starts_with(p)) +} + +/// The paths of a `jobs:read:` scope, or `None` when it spans every job: no +/// path part, or `*` among them. +fn job_read_scope_paths(scope: &ScopeDefinition) -> Option<&[String]> { + scope + .resource + .as_deref() + .filter(|paths| !paths.iter().any(|p| p == "*")) +} + +/// The runnables a token's job reads are confined to: those its `jobs:run` scopes may +/// start, and those its `jobs:read:` scopes name. +pub struct JobReadConfinement { + run: Vec, + read_paths: crate::ScopePathFilter, +} + +impl JobReadConfinement { + /// Whether a job that ran `runnable_path` as `kind` (`scripts`, `flows` or `agents`) + /// is inside the confinement. A read path admits every kind; a run scope only its own. + pub fn admits(&self, kind: &str, runnable_path: &str) -> bool { + if self.read_paths.allows(runnable_path) { + return true; + } + let required = ScopeDefinition::new( + ScopeDomain::Jobs.as_str(), + ScopeAction::Run.as_str(), + Some(kind), + Some(vec![runnable_path.to_string()]), + ); + self.run.iter().any(|scope| scope.includes(&required)) + } + + /// Whether a `jobs:read:` scope is part of the confinement. + pub fn has_read_paths(&self) -> bool { + matches!(&self.read_paths, crate::ScopePathFilter::Restricted { exact, prefix } + if !exact.is_empty() || !prefix.is_empty()) + } +} + +fn push_read_paths(paths: &[String], exact: &mut Vec, prefix: &mut Vec) { + for path in paths { + match path.strip_suffix("/*") { + Some(p) => prefix.push(p.to_string()), + None => exact.push(path.clone()), + } + } +} + +/// The paths a token's job lists are filtered to. Decided by its read grants alone, +/// unlike [`job_read_confinement`]: a `jobs:run` scope grants no enumeration, so even a +/// bare one beside a `jobs:read:` leaves the lists on those paths. Unrestricted +/// for an unscoped token, a `jobs:read` naming no path (or `*`), and `jobs:write`. +pub fn job_list_path_filter(scopes: Option<&[String]>) -> crate::ScopePathFilter { + let mut exact = Vec::new(); + let mut prefix = Vec::new(); + let mut scoped = false; + for scope in scopes + .unwrap_or_default() + .iter() + .filter(|s| !s.starts_with("if_jobs:filter_tags:")) + { + scoped = true; + let Ok(scope) = ScopeDefinition::from_scope_string(scope) else { + continue; + }; + if ScopeDomain::from_str(&scope.domain) != Some(ScopeDomain::Jobs) { + continue; + } + match ScopeAction::from_str(&scope.action) { + Some(ScopeAction::Read) => match job_read_scope_paths(&scope) { + Some(paths) => push_read_paths(paths, &mut exact, &mut prefix), + None => return crate::ScopePathFilter::AllowAll, + }, + Some(ScopeAction::Write) => return crate::ScopePathFilter::AllowAll, + _ => continue, + } + } + if scoped { + crate::ScopePathFilter::Restricted { exact, prefix } + } else { + crate::ScopePathFilter::AllowAll + } +} + +/// What a token's job reads are confined to, or `None` when they are not confined to +/// particular runnables. /// /// A run scope is what the trigger UI mints per script or flow and hands to a webhook /// caller / CI job: it may start the runnables it names and follow those runs, so its -/// by-id job reads must stay within what it can start (enforced by -/// `require_job_read_access`). Both the path (`jobs:run:flows:f/team/etl`) and the -/// kind-only (`jobs:run:scripts`, which legacy `jobs:runscript` tokens carry) forms -/// confine, since `ScopeDefinition::includes` already matches a candidate -/// `jobs:run::` against either. +/// by-id job reads must stay within what it can start. Both the path +/// (`jobs:run:flows:f/team/etl`) and the kind-only (`jobs:run:scripts`, which legacy +/// `jobs:runscript` tokens carry) forms confine, since `ScopeDefinition::includes` +/// already matches a candidate `jobs:run::` against either. +/// +/// A `jobs:read:` scope reads the runs of the runnables at those paths and +/// nothing else. The two kinds of grant add up: a job either admits is readable. /// /// Returns `None` — unconfined — when the token is effectively unscoped, or carries a -/// jobs scope that grants job reads in its own right: `jobs:read`/`jobs:write`, or a -/// bare `jobs:run` (it can start anything, so confining its reads to "what it may run" -/// would restrict nothing). -pub fn job_read_run_confinement(scopes: Option<&[String]>) -> Option> { - let mut confinement = Vec::new(); +/// jobs scope that grants every job read in its own right: a `jobs:read` naming no path +/// (or `*`), a bare `jobs:run` (it can start anything, so confining its reads to "what +/// it may run" would restrict nothing), or `jobs:write`, whose paths are ignored on +/// reads as they are on every job write. +pub fn job_read_confinement(scopes: Option<&[String]>) -> Option { + let mut run = Vec::new(); + let mut exact = Vec::new(); + let mut prefix = Vec::new(); + let mut confined = false; for scope in scopes? .iter() .filter(|s| !s.starts_with("if_jobs:filter_tags:")) @@ -972,26 +688,76 @@ pub fn job_read_run_confinement(scopes: Option<&[String]>) -> Option { - confinement.push(scope) + confined = true; + run.push(scope) } + Some(ScopeAction::Read) => { + let Some(paths) = job_read_scope_paths(&scope) else { + return None; + }; + confined = true; + push_read_paths(paths, &mut exact, &mut prefix); + } + // Grants no reads (see `scope_grants_access`), so it neither confines nor + // frees them. + Some(ScopeAction::Cancel) => continue, Some(_) => return None, None => continue, } } + confined.then_some(JobReadConfinement { + run, + read_paths: crate::ScopePathFilter::Restricted { exact, prefix }, + }) +} + +/// The `jobs:cancel:` scopes a token's cancels are confined to, or `None` when +/// they are not confined: an unscoped token, or one whose cancel grant is not +/// path-scoped (`jobs:cancel`, or `jobs:write`, which stays resource-blind here as on +/// every other job write). +/// +/// A job is within the confinement when its own `runnable_path` or that of any of its +/// `parent_job` ancestors matches (see `cancel_confinement_admits`), so a token scoped +/// to a flow can cancel the flow's steps. +pub fn job_cancel_path_confinement(scopes: Option<&[String]>) -> Option> { + let mut confinement = Vec::new(); + for scope in scopes? + .iter() + .filter(|s| !s.starts_with("if_jobs:filter_tags:")) + { + let Ok(scope) = ScopeDefinition::from_scope_string(scope) else { + continue; + }; + if ScopeDomain::from_str(&scope.domain) != Some(ScopeDomain::Jobs) { + continue; + } + match ScopeAction::from_str(&scope.action) { + Some(ScopeAction::Cancel) if scope.resource.is_some() => confinement.push(scope), + Some(ScopeAction::Cancel | ScopeAction::Write) => return None, + _ => continue, + } + } (!confinement.is_empty()).then_some(confinement) } -/// Whether a job that ran `runnable_path` as `kind` (`scripts` or `flows`) is inside a -/// [`job_read_run_confinement`] set. -pub fn run_confinement_admits( - confinement: &[ScopeDefinition], - kind: &str, - runnable_path: &str, -) -> bool { +/// Whether the token holds a `jobs:cancel` scope, path-scoped or not. +pub fn has_job_cancel_grant(scopes: Option<&[String]>) -> bool { + scopes.is_some_and(|scopes| { + scopes.iter().any(|s| { + ScopeDefinition::from_scope_string(s).is_ok_and(|s| { + ScopeDomain::from_str(&s.domain) == Some(ScopeDomain::Jobs) + && ScopeAction::from_str(&s.action) == Some(ScopeAction::Cancel) + }) + }) + }) +} + +/// Whether a job of `runnable_path` is inside a [`job_cancel_path_confinement`] set. +pub fn cancel_confinement_admits(confinement: &[ScopeDefinition], runnable_path: &str) -> bool { let required = ScopeDefinition::new( ScopeDomain::Jobs.as_str(), - ScopeAction::Run.as_str(), - Some(kind), + ScopeAction::Cancel.as_str(), + None, Some(vec![runnable_path.to_string()]), ); confinement.iter().any(|scope| scope.includes(&required)) @@ -1052,6 +818,17 @@ fn scope_grants_access( })); } + // A path-scoped `jobs:read:` is the one job scope this layer does not treat + // as resource-blind: it reaches only the routes whose handlers confine it to those + // paths, so a job read route is refused to it until it is listed there. + if scope_domain == ScopeDomain::Jobs + && scope_action == ScopeAction::Read + && required_action == ScopeAction::Read + && job_read_scope_paths(scope).is_some() + { + return Ok(route_path.is_some_and(is_read_path_scoped_route)); + } + if !scope_action.includes(&required_action) { return Ok(false); } @@ -1202,6 +979,60 @@ pub fn scope_for_route(method: &str, path: &str) -> Option { }) } +/// Routes the runtime of a job calls about that job alone: progress, its root id, its +/// resume and approval urls, a workflow-as-code checkpoint or task. A job token restricted +/// by `job_token_scopes` keeps these whatever its scopes, or the job could not run at all, +/// but only for its own job: the job id in the path must be the token's. +pub fn is_own_job_runtime_route(route_path: &str, http_method: &str, job_id: uuid::Uuid) -> bool { + let Some(rest) = route_path.strip_prefix("/api/w/") else { + return false; + }; + let Some((_workspace, rest)) = rest.split_once('/') else { + return false; + }; + let segments: Vec<&str> = rest.split('/').collect(); + let (method_ok, id_index) = match segments.as_slice() { + ["jobs_u", "get" | "get_root_job_id", ..] => (http_method == "GET", 2), + ["job_metrics", "set_progress", ..] => (http_method == "POST", 2), + ["job_metrics", "get_progress", ..] => (http_method == "GET", 2), + ["jobs" | "jobs_u", "resume_urls" | "wac_approval_urls", ..] => (http_method == "GET", 2), + ["jobs", "wac", "inline_checkpoint", ..] => (http_method == "POST", 3), + ["jobs", "run", "workflow_as_code", ..] => (http_method == "POST", 3), + _ => return false, + }; + method_ok + && segments + .get(id_index) + .and_then(|id| uuid::Uuid::parse_str(id).ok()) + .is_some_and(|id| id == job_id) +} + +/// Routes reading the state of a flow run: step results (`results.x` in input transforms, +/// loop and branch results) and the run's user state. The flow orchestrator evaluates a +/// step's inputs with the token of the step that just finished, and the SDK reads user state +/// at the root job, so a restricted job token keeps these for every job of its own run. +/// Returns the job id the path names; the caller checks it against the token's lineage. +pub fn flow_run_read_route_job(route_path: &str, http_method: &str) -> Option { + let rest = route_path.strip_prefix("/api/w/")?; + let (_workspace, rest) = rest.split_once('/')?; + let segments: Vec<&str> = rest.split('/').collect(); + let id = match segments.as_slice() { + ["jobs" | "jobs_u", "result_by_id", id, ..] if http_method == "GET" => id, + ["jobs_u", "completed", "get_result" | "get_result_maybe", id, ..] + if http_method == "GET" => + { + id + } + ["jobs" | "jobs_u", "flow", "user_states", id, ..] + if http_method == "GET" || http_method == "POST" => + { + id + } + _ => return None, + }; + uuid::Uuid::parse_str(id).ok() +} + /// Helper function to check if scopes allow access to a route pub fn check_scopes_for_route( token_scopes: Option<&[String]>, @@ -1221,6 +1052,63 @@ pub fn check_scopes_for_route( mod tests { use super::*; + #[test] + fn own_job_runtime_routes_admit_only_the_tokens_job() { + let own = uuid::Uuid::new_v4(); + let other = uuid::Uuid::new_v4(); + let ok = |path: &str, method: &str| is_own_job_runtime_route(path, method, own); + assert!(ok( + &format!("/api/w/ws/job_metrics/set_progress/{own}"), + "POST" + )); + assert!(ok( + &format!("/api/w/ws/jobs_u/get_root_job_id/{own}"), + "GET" + )); + assert!(ok(&format!("/api/w/ws/jobs/resume_urls/{own}/0"), "GET")); + assert!(ok( + &format!("/api/w/ws/jobs/wac/inline_checkpoint/{own}"), + "POST" + )); + assert!(ok( + &format!("/api/w/ws/jobs/run/workflow_as_code/{own}/main"), + "POST" + )); + assert!(!ok( + &format!("/api/w/ws/job_metrics/set_progress/{other}"), + "POST" + )); + assert!(!ok(&format!("/api/w/ws/jobs_u/get/{own}"), "POST")); + assert!(!ok(&format!("/api/w/ws/jobs/run/p/{own}"), "POST")); + assert!(!ok("/api/w/ws/variables/get_value/u/admin/secret", "GET")); + let run = |path: &str, method: &str| flow_run_read_route_job(path, method); + assert_eq!( + run(&format!("/api/w/ws/jobs/result_by_id/{other}/b"), "GET"), + Some(other) + ); + assert_eq!( + run( + &format!("/api/w/ws/jobs_u/completed/get_result/{other}"), + "GET" + ), + Some(other) + ); + assert_eq!( + run( + &format!("/api/w/ws/jobs/flow/user_states/{other}/k"), + "POST" + ), + Some(other) + ); + assert_eq!( + run( + &format!("/api/w/ws/jobs_u/completed/delete/{other}"), + "POST" + ), + None + ); + } + #[test] fn test_scope_definition_parsing() { let scope = ScopeDefinition::from_scope_string("jobs:read").unwrap(); @@ -1395,28 +1283,16 @@ mod tests { #[test] fn run_scopes_confine_job_reads_by_kind_and_path() { let confinement = - job_read_run_confinement(Some(&["jobs:run:flows:f/team/*".to_string()])).unwrap(); - assert!(run_confinement_admits(&confinement, "flows", "f/team/etl")); + job_read_confinement(Some(&["jobs:run:flows:f/team/*".to_string()])).unwrap(); + assert!(confinement.admits("flows", "f/team/etl")); // Right path, wrong kind — a script named like the flow is not the flow. - assert!(!run_confinement_admits( - &confinement, - "scripts", - "f/team/etl" - )); - assert!(!run_confinement_admits( - &confinement, - "flows", - "f/other/etl" - )); + assert!(!confinement.admits("scripts", "f/team/etl")); + assert!(!confinement.admits("flows", "f/other/etl")); // A kind-only scope confines to that kind, at any path. - let kind_only = job_read_run_confinement(Some(&["jobs:run:scripts".to_string()])).unwrap(); - assert!(run_confinement_admits( - &kind_only, - "scripts", - "u/admin/anything" - )); - assert!(!run_confinement_admits(&kind_only, "flows", "f/team/etl")); + let kind_only = job_read_confinement(Some(&["jobs:run:scripts".to_string()])).unwrap(); + assert!(kind_only.admits("scripts", "u/admin/anything")); + assert!(!kind_only.admits("flows", "f/team/etl")); // Scopes that grant job reads in their own right leave reads unconfined. for scopes in [ @@ -1429,11 +1305,111 @@ mod tests { vec!["if_jobs:filter_tags:deno".to_string()], ] { assert!( - job_read_run_confinement(Some(&scopes)).is_none(), + job_read_confinement(Some(&scopes)).is_none(), "{scopes:?} must not confine job reads" ); } - assert!(job_read_run_confinement(None).is_none()); + assert!(job_read_confinement(None).is_none()); + } + + #[test] + fn jobs_read_paths_confine_reads_and_reach_only_confined_routes() { + let scopes = |s: &[&str]| s.iter().map(|s| s.to_string()).collect::>(); + let scoped = scopes(&["jobs:read:f/served/*,u/svc/report"]); + let confinement = job_read_confinement(Some(&scoped)).unwrap(); + // Any kind of runnable at a named path. + assert!(confinement.admits("flows", "f/served/etl")); + assert!(confinement.admits("scripts", "u/svc/report")); + assert!(!confinement.admits("scripts", "f/other/etl")); + assert!(!confinement.admits("scripts", "f/served_not/etl")); + let lists = |s: &[&str], path: &str| job_list_path_filter(Some(&scopes(s))).allows(path); + assert!(lists( + &["jobs:read:f/served/*,u/svc/report"], + "f/served/etl" + )); + assert!(!lists( + &["jobs:read:f/served/*,u/svc/report"], + "f/other/etl" + )); + + // A run scope adds the runnables it may start to the by-id reads, and nothing to + // the lists, not even a bare one that leaves the by-id reads unconfined. + let both = scopes(&["jobs:read:f/served/*", "jobs:run:scripts:f/other/etl"]); + let confinement = job_read_confinement(Some(&both)).unwrap(); + assert!(confinement.admits("scripts", "f/other/etl")); + assert!(!confinement.admits("flows", "f/other/etl")); + for run in ["jobs:run:scripts:f/other/etl", "jobs:run"] { + assert!( + !lists(&["jobs:read:f/served/*", run], "f/other/etl"), + "{run}" + ); + } + for unfiltered in [ + &["jobs:read"][..], + &["jobs:read:f/served/*", "jobs:write"], + &["if_jobs:filter_tags:deno"], + ] { + assert!(lists(unfiltered, "f/other/etl"), "{unfiltered:?}"); + } + + for unconfined in [ + &["jobs:read:*"][..], + &["jobs:read:f/served/*", "jobs:read"], + &["jobs:read:f/served/*", "jobs:write"], + &["jobs:write:f/served/*"], + ] { + assert!( + job_read_confinement(Some(&scopes(unconfined))).is_none(), + "{unconfined:?}" + ); + } + + let id = "0190f4c2-0000-7000-8000-000000000000"; + for route in [ + format!("jobs_u/get/{id}"), + format!("jobs_u/completed/get_result/{id}"), + format!("jobs_u/getupdate_sse/{id}"), + format!("jobs_u/flow/approval_info/{id}"), + "jobs/list".to_string(), + "jobs/completed/list".to_string(), + "jobs/queue/list".to_string(), + ] { + let route = format!("/api/w/ws/{route}"); + assert!( + check_route_access(&scoped, &route, "GET").is_ok(), + "{route}" + ); + } + // Reads that aggregate over the workspace, resolve past the job they name, or + // mint a credential stay with a `jobs:read` that names no path. + for route in [ + "jobs/list_filtered_uuids".to_string(), + "jobs/queue/list_filtered_uuids".to_string(), + "jobs/queue/count".to_string(), + "jobs/completed/count".to_string(), + "jobs/completed/count_jobs".to_string(), + "jobs/completed/export".to_string(), + "jobs/queue/export".to_string(), + "jobs/asset_dispatch_edges".to_string(), + format!("jobs/queue/scheduled_for/{id}"), + format!("jobs/result_by_id/{id}/a"), + format!("jobs/job_view_token/{id}"), + format!("jobs_u/get_root_job_id/{id}"), + ] { + let route = format!("/api/w/ws/{route}"); + assert!( + check_route_access(&scoped, &route, "GET").is_err(), + "{route}" + ); + assert!( + check_route_access(&scopes(&["jobs:read"]), &route, "GET").is_ok(), + "{route}" + ); + assert!( + check_route_access(&scopes(&["jobs:read:*"]), &route, "GET").is_ok(), + "{route}" + ); + } } #[test] @@ -1781,4 +1757,68 @@ mod tests { // Unknown route -> None so the caller fails closed. assert!(scope_for_route("GET", "/healthz").is_none()); } + + #[test] + fn jobs_cancel_grants_only_the_cancel_routes() { + let cancel = vec!["jobs:cancel:f/served/*".to_string()]; + let id = "0190f4c2-0000-7000-8000-000000000000"; + for route in [ + format!("/api/w/ws/jobs_u/queue/cancel/{id}"), + format!("/api/w/ws/jobs_u/queue/force_cancel/{id}"), + "/api/w/ws/jobs_u/queue/cancel_persistent/f/served/s".to_string(), + "/api/w/ws/jobs/queue/cancel_selection".to_string(), + ] { + assert!( + check_route_access(&cancel, &route, "POST").is_ok(), + "{route}" + ); + assert!( + check_route_access(&["jobs:write".to_string()], &route, "POST").is_ok(), + "{route}" + ); + assert!( + check_route_access(&["jobs:read".to_string()], &route, "POST").is_err(), + "{route}" + ); + } + for (route, method) in [ + (format!("/api/w/ws/jobs_u/get/{id}"), "GET"), + ("/api/w/ws/jobs/list".to_string(), "GET"), + (format!("/api/w/ws/jobs/flow/resume_suspended/{id}"), "POST"), + (format!("/api/w/ws/jobs/queue/run_now/{id}"), "POST"), + ("/api/w/ws/jobs/run/p/f/served/s".to_string(), "POST"), + ] { + assert!( + check_route_access(&cancel, &route, method).is_err(), + "{route}" + ); + } + } + + #[test] + fn jobs_cancel_path_confinement() { + let scopes = |s: &[&str]| s.iter().map(|s| s.to_string()).collect::>(); + let conf = + job_cancel_path_confinement(Some(&scopes(&["jobs:cancel:f/served/*,u/svc/kill_me"]))) + .unwrap(); + assert!(cancel_confinement_admits(&conf, "f/served/etl")); + assert!(cancel_confinement_admits(&conf, "u/svc/kill_me")); + assert!(!cancel_confinement_admits(&conf, "f/other/etl")); + assert!(!cancel_confinement_admits(&conf, "f/served_not/etl")); + + // A cancel grant that is not path-scoped leaves cancels unconfined. + for s in [ + &["jobs:cancel"][..], + &["jobs:cancel:f/a/*", "jobs:write"], + &[], + ] { + assert!( + job_cancel_path_confinement(Some(&scopes(s))).is_none(), + "{s:?}" + ); + } + // A cancel scope neither confines nor frees a run token's reads. + let run = scopes(&["jobs:run:flows:f/a/b", "jobs:cancel"]); + assert!(job_read_confinement(Some(&run)).is_some()); + } } diff --git a/backend/windmill-api-flows/src/flows.rs b/backend/windmill-api-flows/src/flows.rs index aa86a7ff07..1007cfd5b1 100644 --- a/backend/windmill-api-flows/src/flows.rs +++ b/backend/windmill-api-flows/src/flows.rs @@ -258,7 +258,8 @@ async fn list_flows( r#"SELECT DISTINCT ON (path) path, value as "value!: sqlx::types::Json>", - created_at + created_at, + email IS NULL as "legacy!" FROM draft WHERE workspace_id = $1 AND typ = 'flow' @@ -315,9 +316,9 @@ async fn list_flows( inherited_labels: None, is_draft: true, draft_path, - // Synthesized rows are the authed user's own draft. + // Owned by nobody when legacy; see scripts.rs. draft_users: Some(sqlx::types::Json(vec![DraftUserRef { - username: Some(authed.username.clone()), + username: (!row.legacy).then(|| authed.username.clone()), }])), }); } @@ -803,6 +804,24 @@ async fn create_flow( check_schedule_conflict(&mut tx, &w_id, &nf.path).await?; let schema_str = nf.schema.and_then(|x| serde_json::to_string(&x.0).ok()); + let job_token_scopes = nf + .job_token_scopes + .as_ref() + .and_then(|scopes| scopes.as_deref()) + .map(windmill_common::scopes::validate_job_token_scopes) + .transpose()?; + let restricts_steps = windmill_common::scopes::validate_flow_step_job_token_scopes( + &serde_json::from_str::(nf.value.get()) + .map_err(|e| windmill_common::error::Error::BadRequest(e.to_string()))?, + )?; + if job_token_scopes.is_some() || restricts_steps { + windmill_common::min_version::MIN_VERSION_SUPPORTS_JOB_TOKEN_SCOPES + .assert() + .await?; + } + if restricts_steps { + windmill_common::feature_usage::log_feature_usage("job_token_scopes", "deploy", "step:set"); + } let resolved_on_behalf_of = windmill_common::resolve_on_behalf_of( nf.on_behalf_of_email.as_deref(), nf.on_behalf_of.as_deref(), @@ -823,14 +842,14 @@ async fn create_flow( dedicated_worker, visible_to_runner_only, ws_error_handler_muted, value, schema, edited_by, edited_at, labels, - on_behalf_of, on_behalf_of_email + on_behalf_of, on_behalf_of_email, job_token_scopes ) VALUES ( $1, $2, $3, $4, NULL, '', $5, $6, $7, $8, $9, $10::text::json, $11, now(), $12, - $13, $14 + $13, $14, $15 )"#, w_id, nf.path, @@ -846,9 +865,11 @@ async fn create_flow( nf.labels.as_deref() as Option<&[String]>, resolved_on_behalf_of, legacy_on_behalf_of_email, + job_token_scopes.as_deref() as Option<&[String]>, ) .execute(&mut *tx) .await?; + windmill_common::scopes::log_job_token_scopes_deploy("flow", job_token_scopes.as_deref()); let version = sqlx::query_scalar!( "INSERT INTO flow_version (workspace_id, path, value, schema, created_by) @@ -973,6 +994,7 @@ async fn create_flow( None, None, None, + None, ) .await?; @@ -1152,7 +1174,7 @@ async fn get_flow_version( let mut tx = user_db.begin(&authed).await?; let flow = sqlx::query_as::<_, Flow>( - "SELECT flow.workspace_id, flow.path, flow.summary, flow.description, flow.archived, flow.extra_perms, flow.dedicated_worker, flow.tag, flow.ws_error_handler_muted, flow.timeout, flow.visible_to_runner_only, flow.on_behalf_of, flow.labels, flow_version.schema, flow_version.value, flow_version.created_at as edited_at, flow_version.created_by as edited_by + "SELECT flow.workspace_id, flow.path, flow.summary, flow.description, flow.archived, flow.extra_perms, flow.dedicated_worker, flow.tag, flow.ws_error_handler_muted, flow.timeout, flow.visible_to_runner_only, flow.on_behalf_of, flow.labels, flow.job_token_scopes, flow_version.schema, flow_version.value, flow_version.created_at as edited_at, flow_version.created_by as edited_by FROM flow LEFT JOIN flow_version ON flow_version.path = flow.path AND flow_version.workspace_id = flow.workspace_id WHERE flow.path = $1 AND flow.workspace_id = $2 AND flow_version.id = $3", @@ -1212,6 +1234,7 @@ async fn get_flow_version_by_id( flow.visible_to_runner_only, flow.on_behalf_of, flow.labels, + flow.job_token_scopes, flow_version.schema, flow_version.value, flow_version.created_at as edited_at, @@ -1367,6 +1390,27 @@ async fn update_flow( let old_dep_job = not_found_if_none(old_dep_job, "Flow", flow_path)?; let is_new_path = nf.path != flow_path; let schema_str = schema.and_then(|x| serde_json::to_string(&x).ok()); + // Absent keeps the deployed value: a client unaware of the setting must not drop a + // restriction by saving the flow. + let set_job_token_scopes = nf.job_token_scopes.is_some(); + let job_token_scopes = nf + .job_token_scopes + .as_ref() + .and_then(|scopes| scopes.as_deref()) + .map(windmill_common::scopes::validate_job_token_scopes) + .transpose()?; + let restricts_steps = windmill_common::scopes::validate_flow_step_job_token_scopes( + &serde_json::from_str::(nf.value.get()) + .map_err(|e| windmill_common::error::Error::BadRequest(e.to_string()))?, + )?; + if job_token_scopes.is_some() || restricts_steps { + windmill_common::min_version::MIN_VERSION_SUPPORTS_JOB_TOKEN_SCOPES + .assert() + .await?; + } + if restricts_steps { + windmill_common::feature_usage::log_feature_usage("job_token_scopes", "deploy", "step:set"); + } let resolved_on_behalf_of = windmill_common::resolve_on_behalf_of( nf.on_behalf_of_email.as_deref(), nf.on_behalf_of.as_deref(), @@ -1401,7 +1445,8 @@ async fn update_flow( edited_at = now(), labels = COALESCE($13, labels), on_behalf_of = $14, - on_behalf_of_email = $15 + on_behalf_of_email = $15, + job_token_scopes = CASE WHEN $16 THEN $17 ELSE job_token_scopes END WHERE path = $11 AND workspace_id = $12", if is_new_path { flow_path } else { &nf.path }, @@ -1419,19 +1464,24 @@ async fn update_flow( nf.labels.as_deref() as Option<&[String]>, resolved_on_behalf_of, legacy_on_behalf_of_email, + set_job_token_scopes, + job_token_scopes.as_deref() as Option<&[String]>, ) .execute(&mut *tx) .await .map_err(|e| { error::Error::internal_err(format!("Error updating flow due to flow update: {e:#}")) })?; + if set_job_token_scopes { + windmill_common::scopes::log_job_token_scopes_deploy("flow", job_token_scopes.as_deref()); + } if is_new_path { // if new path, must clone flow to new path and delete old flow for flow_version foreign key constraint sqlx::query!( "INSERT INTO flow - (workspace_id, path, summary, description, archived, extra_perms, dependency_job, tag, ws_error_handler_muted, dedicated_worker, timeout, visible_to_runner_only, on_behalf_of, on_behalf_of_email, concurrency_key, versions, value, schema, edited_by, edited_at, labels) - SELECT workspace_id, $1, summary, description, archived, extra_perms, dependency_job, tag, ws_error_handler_muted, dedicated_worker, timeout, visible_to_runner_only, on_behalf_of, on_behalf_of_email, concurrency_key, versions, value, schema, edited_by, edited_at, labels + (workspace_id, path, summary, description, archived, extra_perms, dependency_job, tag, ws_error_handler_muted, dedicated_worker, timeout, visible_to_runner_only, on_behalf_of, on_behalf_of_email, concurrency_key, versions, value, schema, edited_by, edited_at, labels, job_token_scopes) + SELECT workspace_id, $1, summary, description, archived, extra_perms, dependency_job, tag, ws_error_handler_muted, dedicated_worker, timeout, visible_to_runner_only, on_behalf_of, on_behalf_of_email, concurrency_key, versions, value, schema, edited_by, edited_at, labels, job_token_scopes FROM flow WHERE path = $2 AND workspace_id = $3", nf.path, @@ -1695,6 +1745,7 @@ async fn update_flow( None, None, None, + None, ) .await?; @@ -1870,6 +1921,7 @@ async fn get_flow_by_path( flow.visible_to_runner_only, flow.on_behalf_of, flow.labels, + flow.job_token_scopes, folder_labels(flow.workspace_id, flow.path) AS inherited_labels, flow_version.id AS version_id, flow_version.schema, @@ -1911,6 +1963,7 @@ async fn get_flow_by_path( flow.visible_to_runner_only, flow.on_behalf_of, flow.labels, + flow.job_token_scopes, folder_labels(flow.workspace_id, flow.path) AS inherited_labels, flow_version.id AS version_id, flow_version.schema, @@ -2335,6 +2388,7 @@ mod tests { apply_preprocessor: None, pass_flow_input_directly: None, debouncing: None, + job_token_scopes: None, }, FlowModule { id: "b".to_string(), @@ -2370,6 +2424,7 @@ mod tests { apply_preprocessor: None, pass_flow_input_directly: None, debouncing: None, + job_token_scopes: None, }, FlowModule { id: "c".to_string(), @@ -2405,6 +2460,7 @@ mod tests { apply_preprocessor: None, pass_flow_input_directly: None, debouncing: None, + job_token_scopes: None, }, ], failure_module: Some(Box::new(FlowModule { @@ -2439,6 +2495,7 @@ mod tests { apply_preprocessor: None, pass_flow_input_directly: None, debouncing: None, + job_token_scopes: None, })), preprocessor_module: None, same_worker: false, diff --git a/backend/windmill-api-integration-tests/tests/settings.rs b/backend/windmill-api-integration-tests/tests/settings.rs index 348d945a86..7fd60ed8e1 100644 --- a/backend/windmill-api-integration-tests/tests/settings.rs +++ b/backend/windmill-api-integration-tests/tests/settings.rs @@ -200,3 +200,81 @@ async fn test_alert_job_queue_waiting_in_global_settings(db: Pool) -> Ok(()) } + +async fn read_secret_surfaces( + base: &str, + names: &[&str], +) -> anyhow::Result> { + let mut paths = vec![ + "instance_config".to_string(), + "instance_config/yaml".to_string(), + ]; + #[cfg(feature = "enterprise")] + paths.push("list_global".to_string()); + paths.extend(names.iter().map(|n| format!("global/{n}"))); + let mut out = vec![]; + for path in paths { + let resp = authed(client().get(format!("{base}/{path}"))) + .send() + .await?; + let status = resp.status().as_u16(); + out.push((path, status, resp.text().await?)); + } + Ok(out) +} + +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn test_server_secrets_follow_export_flag(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let secrets = [ + ("jwt_secret", json!("planted-jwt-secret")), + ("rsa_keys", json!({"private_key": "planted-rsa-key"})), + ( + "custom_instance_replication_pwd", + json!("planted-replication-pwd"), + ), + ( + "external_instance_pg_state", + json!({"admin_pwd": "planted-pg-state"}), + ), + ]; + for (name, value) in &secrets { + sqlx::query( + "INSERT INTO global_settings (name, value) VALUES ($1, $2) + ON CONFLICT (name) DO UPDATE SET value = EXCLUDED.value", + ) + .bind(name) + .bind(value) + .execute(&db) + .await?; + } + let names: Vec<&str> = secrets.iter().map(|(n, _)| *n).collect(); + let server = ApiServer::start(db.clone()).await?; + let base = format!("http://localhost:{}/api/settings", server.addr.port()); + + // Default: a full export, so `get-config` and `pull` can migrate an instance. + std::env::remove_var("EXPORT_SERVER_SECRETS"); + for (path, status, body) in read_secret_surfaces(&base, &names).await? { + assert_2xx(status, &body, &path); + assert!( + body.contains("planted-"), + "{path} dropped a server secret: {body}" + ); + } + + std::env::set_var("EXPORT_SERVER_SECRETS", "0"); + let withheld = read_secret_surfaces(&base, &names).await; + std::env::remove_var("EXPORT_SERVER_SECRETS"); + for (path, status, body) in withheld? { + if path.starts_with("global/") { + assert_eq!(status, 400, "GET {path} returned {status}"); + } else { + assert_2xx(status, &body, &path); + } + assert!( + !body.contains("planted-"), + "{path} returned a server secret: {body}" + ); + } + Ok(()) +} diff --git a/backend/windmill-api-integration-tests/tests/workspace_comparison.rs b/backend/windmill-api-integration-tests/tests/workspace_comparison.rs index 11c731d75c..1aa2bc8df2 100644 --- a/backend/windmill-api-integration-tests/tests/workspace_comparison.rs +++ b/backend/windmill-api-integration-tests/tests/workspace_comparison.rs @@ -756,6 +756,13 @@ async fn test_compare_workspaces_trigger_and_schedule(db: Pool) -> any ) .execute(&db) .await?; + // The scheduler's missed-occurrence counters only ever move on the enabled side. + sqlx::query( + "UPDATE schedule SET late_run_streak = 1, missed_occurrences = 2, last_missed_at = NOW() + WHERE workspace_id = 'test-workspace' AND path = 'f/sch/runtime_only'", + ) + .execute(&db) + .await?; // ------ Schedule: config change (script_path) in fork. Should diff. sqlx::query!( @@ -2537,7 +2544,10 @@ async fn test_rename_records_the_vacated_path(db: Pool) -> anyhow::Res .await?; let status = resp.status(); let hash = resp.text().await?; - assert!(status.is_success(), "script deploy failed: {status} — {hash}"); + assert!( + status.is_success(), + "script deploy failed: {status} — {hash}" + ); parent_hash = Some(hash.trim().trim_matches('"').to_string()); } let mut vacated = None; diff --git a/backend/windmill-api-jobs/src/execution.rs b/backend/windmill-api-jobs/src/execution.rs index 1a750c38ba..c35b9e41b0 100644 --- a/backend/windmill-api-jobs/src/execution.rs +++ b/backend/windmill-api-jobs/src/execution.rs @@ -124,7 +124,9 @@ pub async fn drop_unclaimable_run_lineage( /// The jobs of `referenced` that `authed` cannot claim as its own run lineage: anything but the /// token's own job and that job's `parent_job`, `root_job` and `flow_innermost_root_job`, or for -/// a workspace admin anything outside the workspace. +/// a workspace admin anything outside the workspace. A restricted job token never gets the admin +/// latitude: its flow-run routes trust this lineage (`job_in_same_flow_run`), so a claimed +/// unrelated run would escape its scopes. pub async fn unclaimable_run_lineage( db: &DB, w_id: &str, @@ -157,7 +159,8 @@ pub async fn unclaimable_run_lineage( if referenced.is_empty() { return Ok(referenced); } - let in_workspace = if authed.is_admin { + let restricted_job_token = authed.job_id.is_some() && authed.scopes.is_some(); + let in_workspace = if authed.is_admin && !restricted_job_token { sqlx::query_scalar!( "SELECT id FROM v2_job WHERE id = ANY($1) AND workspace_id = $2", &referenced, @@ -825,6 +828,7 @@ pub async fn handle_chat_conversation_messages( tx, memory_id, Some(job_id), + job_id, &user_message, MessageType::User, None, @@ -865,6 +869,7 @@ pub async fn run_flow<'c>( chat_input_enabled, early_return, labels, + job_token_scopes, .. } = flow_version_info; @@ -914,6 +919,7 @@ pub async fn run_flow<'c>( ) }; + let scope_ceiling = windmill_api_auth::caller_scope_ceiling(db, authed).await?; let (uuid, mut tx) = push( &db, tx, @@ -924,6 +930,7 @@ pub async fn run_flow<'c>( version, apply_preprocessor, labels, + job_token_scopes, }, push_args, authed.display_username(), @@ -950,6 +957,7 @@ pub async fn run_flow<'c>( None, authed.trigger_or_fallback(trigger), run_query.suspended_mode, + scope_ceiling.as_deref(), ) .await?; @@ -1141,6 +1149,7 @@ pub async fn push_script_job_by_path_into_queue<'c>( ) }; + let scope_ceiling = windmill_api_auth::caller_scope_ceiling(&db, &authed).await?; let (uuid, tx) = push( &db, tx, @@ -1176,6 +1185,7 @@ pub async fn push_script_job_by_path_into_queue<'c>( None, authed.trigger_or_fallback(trigger), run_query.suspended_mode, + scope_ceiling.as_deref(), ) .await?; diff --git a/backend/windmill-api-jobs/src/job_metrics.rs b/backend/windmill-api-jobs/src/job_metrics.rs index abb95693f5..ddbe5a047a 100644 --- a/backend/windmill-api-jobs/src/job_metrics.rs +++ b/backend/windmill-api-jobs/src/job_metrics.rs @@ -173,10 +173,33 @@ struct JobProgressSetRequest { } async fn set_job_progress( + authed: windmill_api_auth::ApiAuthed, Extension(db): Extension, Path((w_id, job_id)): Path<(String, Uuid)>, Json(JobProgressSetRequest { percent, flow_job_id }): Json, ) -> error::JsonResult<()> { + // A restricted job token reaches this route without a scope only for its own job + // (`is_own_job_runtime_route`), so the flow it reports progress to must be its own too. + if let (Some(flow_job_id), Some(token_job), Some(_)) = + (flow_job_id, authed.job_id, &authed.scopes) + { + let own_flow = sqlx::query_scalar!( + "SELECT $2 IN (parent_job, root_job, flow_innermost_root_job) FROM v2_job + WHERE id = $1 AND workspace_id = $3", + token_job, + flow_job_id, + &w_id + ) + .fetch_optional(&db) + .await? + .flatten() + .unwrap_or(false); + if !own_flow { + return Err(error::Error::PermissionDenied(format!( + "flow job {flow_job_id} is not this job's flow" + ))); + } + } // If flow_job_id exists, than we should modify flow_status of corresponding module // Individual jobs and flows are handled differently if let Some(flow_job_id) = flow_job_id { diff --git a/backend/windmill-api-jobs/src/jobs_export.rs b/backend/windmill-api-jobs/src/jobs_export.rs index 6389134031..b2b69367bc 100644 --- a/backend/windmill-api-jobs/src/jobs_export.rs +++ b/backend/windmill-api-jobs/src/jobs_export.rs @@ -112,6 +112,8 @@ pub struct ExportableQueuedJob { pub preprocessed: Option, pub args: Option>>, pub labels: Option>, + #[serde(default)] + pub job_token_scopes: Option>, pub pre_run_error: Option, // v2_job_queue columns (excluding workspace_id and id/created_at/tag/priority) @@ -276,6 +278,7 @@ pub async fn export_queued_jobs( v2_job.preprocessed, v2_job.args as "args: _", v2_job.labels, + job_perms.job_token_scopes as "job_token_scopes?", v2_job.pre_run_error, v2_job_queue.started_at, @@ -298,6 +301,7 @@ pub async fn export_queued_jobs( concurrency_key.key as "concurrency_key?" FROM v2_job_queue INNER JOIN v2_job ON v2_job.id = v2_job_queue.id + LEFT JOIN job_perms ON job_perms.job_id = v2_job_queue.id LEFT JOIN v2_job_runtime ON v2_job_runtime.id = v2_job_queue.id LEFT JOIN v2_job_status ON v2_job_status.id = v2_job_queue.id LEFT JOIN concurrency_key ON concurrency_key.job_id = v2_job_queue.id @@ -445,6 +449,23 @@ pub async fn import_queued_jobs( )); } + // Imported rows bypass `push`, so nothing would cap their tokens: a restricted job token + // could otherwise queue arbitrary jobs that run with its owner's full permissions. + if authed.job_id.is_some() && authed.scopes.is_some() { + return Err(error::Error::PermissionDenied( + "A restricted job token cannot import queued jobs".to_string(), + )); + } + // An imported job gets no `job_perms` row, so its token would be minted without the + // restriction it was queued with. + if let Some(job) = jobs.iter().find(|job| job.job_token_scopes.is_some()) { + return Err(error::Error::BadRequest(format!( + "Queued job {} restricts its job token and cannot be imported; run it again on \ + this instance instead", + job.id + ))); + } + let mut tx = user_db.begin(&authed).await?; for job in jobs { @@ -697,7 +718,8 @@ pub async fn delete_jobs( let mut conversation_ids: Vec = sqlx::query_scalar!( "DELETE FROM flow_conversation_message m USING flow_conversation c - WHERE m.conversation_id = c.id AND c.workspace_id = $1 AND m.job_id = ANY($2) + WHERE m.conversation_id = c.id AND c.workspace_id = $1 + AND COALESCE(m.turn_job_id, m.job_id) = ANY($2) RETURNING m.conversation_id", &w_id, &job_ids diff --git a/backend/windmill-api-jobs/src/query.rs b/backend/windmill-api-jobs/src/query.rs index 857b2e02e2..96d8ca3698 100644 --- a/backend/windmill-api-jobs/src/query.rs +++ b/backend/windmill-api-jobs/src/query.rs @@ -26,6 +26,34 @@ fn not_in_nullable(col: &str, quoted: &[String]) -> String { ) } +/// Narrows a job list to the top-level runs of deployed scripts and flows at an `exact` +/// path, or at or under a `prefix` on the `/` boundary (`ScopePathFilter::allows`). +/// +/// Top-level only because every `runnable_path` index on `v2_job` is partial on +/// `parent_job IS NULL`. Deployed kinds only because a preview's `runnable_path` is +/// whatever its caller sent. +pub fn and_where_root_run_at_paths(sqlb: &mut SqlBuilder, exact: &[String], prefix: &[String]) { + let mut clauses = Vec::new(); + let equal: Vec<_> = exact.iter().chain(prefix).map(|p| quote(p)).collect(); + if !equal.is_empty() { + clauses.push(format!("v2_job.runnable_path IN ({})", equal.join(", "))); + } + for p in prefix { + let pattern = format!("{}/%", escape_ilike_pattern(p)); + clauses.push(format!("v2_job.runnable_path LIKE {}", quote(&pattern))); + } + if clauses.is_empty() { + sqlb.and_where("false"); + return; + } + sqlb.and_where_is_null("v2_job.parent_job") + .and_where( + "v2_job.kind IN ('script', 'script_hub', 'unassigned_script', 'flow', \ + 'unassigned_flow', 'singlestepflow', 'unassigned_singlestepflow')", + ) + .and_where(format!("({})", clauses.join(" OR "))); +} + pub fn filter_list_queue_query( mut sqlb: SqlBuilder, lq: &ListQueueQuery, diff --git a/backend/windmill-api-scripts/src/scripts.rs b/backend/windmill-api-scripts/src/scripts.rs index bce67755bb..f622c4bb91 100644 --- a/backend/windmill-api-scripts/src/scripts.rs +++ b/backend/windmill-api-scripts/src/scripts.rs @@ -382,7 +382,8 @@ async fn list_scripts( r#"SELECT DISTINCT ON (path) path, value as "value!: sqlx::types::Json>", - created_at + created_at, + email IS NULL as "legacy!" FROM draft WHERE workspace_id = $1 AND typ = 'script' @@ -463,9 +464,10 @@ async fn list_scripts( inherited_labels: None, is_draft: true, draft_path, - // Synthesized rows are the authed user's own draft (single-user case). + // A legacy (`email IS NULL`) row belongs to nobody: naming the caller would + // route every discard to their own (absent) row and leave it undeletable. draft_users: Some(sqlx::types::Json(vec![DraftUserRef { - username: Some(authed.username.clone()), + username: (!row.legacy).then(|| authed.username.clone()), }])), }); } @@ -897,6 +899,7 @@ async fn is_noop_deploy_against_parent( ns: &NewScript, parent: &Script, resolved_on_behalf_of: Option<&str>, + resolved_job_token_scopes: Option<&[String]>, db: &DB, ) -> Result { if parent.archived || parent.deleted { @@ -950,6 +953,8 @@ async fn is_noop_deploy_against_parent( // caller-intent flag (auto-resolve parent), not script state auto_parent: _, labels, + // resolved against the deployed value into `resolved_job_token_scopes`, compared below + job_token_scopes: _, // caller-intent flag (preserve user drafts on CLI/git-sync deploys); // transient, never persisted, does not change what the script *is* skip_draft_deletion: _, @@ -1023,6 +1028,9 @@ async fn is_noop_deploy_against_parent( if resolved_on_behalf_of != parent.on_behalf_of.as_deref() { return Ok(false); } + if resolved_job_token_scopes != parent.job_token_scopes.as_deref() { + return Ok(false); + } // Both of a dbt script's derived fields are compared as they WOULD BE STORED, // not as they arrived: the schema comes from the descriptor and the clients // cannot derive it (`windmill-parser-wasm` has no dbt arm), so they send the @@ -1490,6 +1498,37 @@ async fn create_script_internal<'c>( parent_adopted_from_retired_path = ns.parent_hash.is_some(); } + // Absent keeps the previous version's value, read once the parent is settled (a rename + // adopts its source head above), so a client unaware of the setting cannot drop a + // restriction by redeploying or renaming. + let resolved_job_token_scopes: Option> = match (&ns.job_token_scopes, &ns.parent_hash) { + (Some(Some(scopes)), _) => { + windmill_common::min_version::MIN_VERSION_SUPPORTS_JOB_TOKEN_SCOPES + .assert() + .await?; + Some(windmill_common::scopes::validate_job_token_scopes(scopes)?) + } + (Some(None), _) => None, + (None, Some(parent_hash)) => sqlx::query_scalar!( + "SELECT job_token_scopes FROM script WHERE hash = $1 AND workspace_id = $2", + parent_hash.0, + &w_id + ) + .fetch_optional(&db) + .await? + .flatten(), + (None, None) => sqlx::query_scalar!( + "SELECT job_token_scopes FROM script WHERE path = $1 AND workspace_id = $2 \ + AND deleted = false ORDER BY created_at DESC LIMIT 1", + &ns.path, + &w_id + ) + .fetch_optional(&db) + .await? + .flatten(), + }; + ns.job_token_scopes = Some(resolved_job_token_scopes.clone()); + // Before hashing, so the hash and the no-op check see the schema that gets stored. // `{}` counts as absent: an agent filling every tool argument sends it for "none". let schema_absent = ns.schema.as_ref().is_none_or(|s| { @@ -1618,8 +1657,14 @@ async fn create_script_internal<'c>( // CLI pushes must not produce phantom commits on the downstream // git repository. if skip_if_noop - && is_noop_deploy_against_parent(&ns, &ps, resolved_on_behalf_of.as_deref(), &db) - .await? + && is_noop_deploy_against_parent( + &ns, + &ps, + resolved_on_behalf_of.as_deref(), + resolved_job_token_scopes.as_deref(), + &db, + ) + .await? { tracing::info!( workspace_id = %w_id, @@ -2195,8 +2240,8 @@ async fn create_script_internal<'c>( content, created_by, schema, is_template, extra_perms, lock, language, kind, tag, \ envs, concurrent_limit, concurrency_time_window_s, cache_ttl, \ dedicated_worker, ws_error_handler_muted, priority, restart_unless_cancelled, \ - delete_after_use, delete_after_secs, timeout, concurrency_key, visible_to_runner_only, auto_kind, codebase, has_preprocessor, schema_validation, assets, debounce_key, debounce_delay_s, cache_ignore_s3_path, runnable_settings_handle, modules, labels, on_behalf_of, on_behalf_of_email) \ - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9::text::json, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26, $27, $28, $29, $30, $31, $32, $33, $34, $35, $36, $37, $38, $39, $40, $41)", + delete_after_use, delete_after_secs, timeout, concurrency_key, visible_to_runner_only, auto_kind, codebase, has_preprocessor, schema_validation, assets, debounce_key, debounce_delay_s, cache_ignore_s3_path, runnable_settings_handle, modules, labels, on_behalf_of, on_behalf_of_email, job_token_scopes) \ + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9::text::json, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26, $27, $28, $29, $30, $31, $32, $33, $34, $35, $36, $37, $38, $39, $40, $41, $42)", &w_id, &hash.0, ns.path, @@ -2240,9 +2285,14 @@ async fn create_script_internal<'c>( ns.labels.as_deref() as Option<&[String]>, resolved_on_behalf_of, legacy_on_behalf_of_email, + resolved_job_token_scopes.as_deref() as Option<&[String]>, ) .execute(&mut *tx) .await?; + windmill_common::scopes::log_job_token_scopes_deploy( + "script", + resolved_job_token_scopes.as_deref(), + ); // A lock that is not left to a dependency job queues none, so this is the only place its hash // can be recorded. `try_skip_relock` treats a missing hash for an imported script as changed, @@ -2952,6 +3002,7 @@ async fn create_script_internal<'c>( None, None, None, + None, ) .await?; @@ -3075,6 +3126,7 @@ async fn create_script_internal<'c>( None, None, None, + None, ) .await?; tracing::info!("pushed auto-build binary job {job_id} for {script_path}"); diff --git a/backend/windmill-api-settings/src/lib.rs b/backend/windmill-api-settings/src/lib.rs index 71cbcc8460..7e2ec536c0 100644 --- a/backend/windmill-api-settings/src/lib.rs +++ b/backend/windmill-api-settings/src/lib.rs @@ -1235,7 +1235,7 @@ async fn get_instance_config( authed: ApiAuthed, ) -> JsonResult { require_super_admin(&db, &authed).await?; - let config = InstanceConfig::from_db(&db) + let config = InstanceConfig::from_db_for_api(&db) .await .map_err(|e| error::Error::internal_err(e.to_string()))?; Ok(Json(config)) @@ -1246,7 +1246,7 @@ async fn get_instance_config_yaml( authed: ApiAuthed, ) -> error::Result { require_super_admin(&db, &authed).await?; - let config = InstanceConfig::from_db(&db) + let config = InstanceConfig::from_db_for_api(&db) .await .map_err(|e| error::Error::internal_err(e.to_string()))?; let yaml = config @@ -1386,6 +1386,11 @@ pub async fn get_global_setting( { require_super_admin(&db, &authed).await?; } + if instance_config::is_withheld_server_secret(&key) { + return Err(error::Error::BadRequest(format!( + "{key} is a server secret and this server does not export it (EXPORT_SERVER_SECRETS=false)" + ))); + } let value = sqlx::query!("SELECT value FROM global_settings WHERE name = $1", key) .fetch_optional(&db) .await? @@ -1428,7 +1433,10 @@ async fn list_global_settings( require_super_admin(&db, &authed).await?; let settings = sqlx::query_as!(GlobalSetting, "SELECT name, value FROM global_settings") .fetch_all(&db) - .await?; + .await? + .into_iter() + .filter(|s| !instance_config::is_withheld_server_secret(&s.name)) + .collect(); Ok(Json(settings)) } diff --git a/backend/windmill-api-workspaces/src/datatable_migrations.rs b/backend/windmill-api-workspaces/src/datatable_migrations.rs index f762fda175..e63e69d4d5 100644 --- a/backend/windmill-api-workspaces/src/datatable_migrations.rs +++ b/backend/windmill-api-workspaces/src/datatable_migrations.rs @@ -17,6 +17,7 @@ use crate::workspaces::{ use axum::{ extract::{Extension, Path, Query}, + http::StatusCode, routing::{delete, get, post}, Json, Router, }; @@ -235,6 +236,7 @@ async fn run_datatable_migration_job( args.insert("database".to_string(), database_arg.clone()); let push_args = PushArgs { extra: None, args: &args }; + let scope_ceiling = windmill_api_auth::caller_scope_ceiling(db, authed).await?; let (uuid, mut tx) = push( db, PushIsolationLevel::Isolated(user_db.clone(), authed.clone().into()), @@ -278,6 +280,7 @@ async fn run_datatable_migration_job( None, None, None, + scope_ceiling.as_deref(), ) .await?; @@ -401,6 +404,12 @@ async fn connection_identity(client: &tokio_postgres::Client) -> Option<(String, }) } +const MIGRATION_LOCK_KEY: &str = "hashtext('windmill_datatable_migrations')::int8"; + +/// How long a run, rollback or edit waits for the one ahead of it. A migration may run for +/// longer than any request should wait, so past this the caller is told to retry. +const MIGRATION_LOCK_WAIT_SECS: u64 = 30; + /// Open a connection to a data table's own database and hold the session-level /// advisory lock that serializes migration runs/rollbacks. The lock is released /// when the returned client is dropped, so callers must keep it in scope for the @@ -426,18 +435,85 @@ async fn lock_datatable_migration_runs( tracing::error!("Datatable connection error: {}", e); } }); - client - .batch_execute("SELECT pg_advisory_lock(hashtext('windmill_datatable_migrations')::int8)") - .await - .map_err(|e| { - Error::internal_err(format!( - "Failed to acquire migration lock: {}", - pg_error_message(&e) - )) - })?; + hold_migration_lock(&client, datatable_name, MIGRATION_LOCK_WAIT_SECS).await?; Ok(client) } +async fn hold_migration_lock( + client: &tokio_postgres::Client, + datatable_name: &str, + wait_secs: u64, +) -> Result<()> { + // A session whose client vanished without closing the socket otherwise keeps the lock for + // the server's keepalive default, two hours on Linux. `SET` rather than startup `options`, + // which PgBouncer rejects. Behind a pooler they only cover the pooler's own connection to + // the server: there, as on a server refusing them, the bounded wait below is the recovery. + if let Err(e) = client + .batch_execute( + "SET tcp_keepalives_idle = 60; SET tcp_keepalives_interval = 10; \ + SET tcp_keepalives_count = 6", + ) + .await + { + tracing::debug!( + "Could not set server keepalives on the migration lock connection: {}", + pg_error_message(&e) + ); + } + let locked = client + .batch_execute(&format!( + "SET lock_timeout = '{wait_secs}s'; \ + SELECT pg_advisory_lock({MIGRATION_LOCK_KEY}); \ + RESET lock_timeout" + )) + .await; + match locked { + Ok(()) => Ok(()), + Err(e) if e.code() == Some(&SqlState::LOCK_NOT_AVAILABLE) => { + // The key as `pg_locks` splits it: high half in `classid`, low half in `objid`. + let holder = client + .query_opt( + &format!( + "SELECT pid FROM pg_locks + WHERE locktype = 'advisory' AND granted AND objsubid = 1 + AND database = (SELECT oid FROM pg_database WHERE datname = current_database()) + AND ((classid::bigint << 32) | objid::bigint) = {MIGRATION_LOCK_KEY}" + ), + &[], + ) + .await + .ok() + .flatten() + .map(|row| row.get::<_, i32>(0)); + let held_by = match holder { + Some(pid) => format!( + " The lock is held by backend pid {pid}: if nothing is running, a lost \ + connection left that session behind, and \ + `SELECT pg_terminate_backend({pid})` on the data table's database releases it." + ), + None => String::new(), + }; + Err(Error::Generic( + StatusCode::CONFLICT, + format!( + "Another migration run, rollback or edit on the database of data table \ + '{datatable_name}' was still in progress after {wait_secs}s. \ + Retry once it is done.{held_by}" + ), + )) + } + Err(e) => Err(Error::internal_err(format!( + "Failed to acquire migration lock: {}", + pg_error_message(&e) + ))), + } +} + +/// Whether `e` is the lock wait running out, as opposed to the database being unreachable. +fn is_migration_lock_busy(e: &Error) -> bool { + matches!(e, Error::Generic(StatusCode::CONFLICT, _)) +} + /// Read the versions recorded as applied in a data table's `_wm_migrations`, /// scoped to that data table, using an existing connection. An absent table /// (`42P01`) means nothing has been migrated yet. @@ -1358,7 +1434,13 @@ async fn delete_datatable_migration( }; let lock_client = lock_datatable_migration_runs(&db, &w_id, &datatable_name) .await - .map_err(unreachable)?; + .map_err(|e| { + if is_migration_lock_busy(&e) { + e + } else { + unreachable(e) + } + })?; let applied = read_applied_versions_on_client(&lock_client, &datatable_name) .await .map_err(unreachable)?; @@ -1533,7 +1615,13 @@ async fn upsert_datatable_migration( }; let client = lock_datatable_migration_runs(&db, &w_id, &datatable_name) .await - .map_err(unreachable)?; + .map_err(|e| { + if is_migration_lock_busy(&e) { + e + } else { + unreachable(e) + } + })?; let applied = read_applied_versions_on_client(&client, &datatable_name) .await .map_err(unreachable)?; @@ -2041,6 +2129,32 @@ pub(crate) async fn clone_datatable_migrations( mod tests { use super::*; + // The holder is found by reassembling the key from the two halves `pg_locks` shows. + #[sqlx::test(migrations = false)] + async fn a_held_migration_lock_is_reported_with_its_holder(pool: DB) { + let mut config: tokio_postgres::Config = + std::env::var("DATABASE_URL").unwrap().parse().unwrap(); + config.dbname(pool.connect_options().get_database().unwrap()); + let (holder, connection) = config.connect(tokio_postgres::NoTls).await.unwrap(); + tokio::spawn(connection); + let (waiter, connection) = config.connect(tokio_postgres::NoTls).await.unwrap(); + tokio::spawn(connection); + hold_migration_lock(&holder, "main", 1).await.unwrap(); + let holder_pid = holder + .query_one("SELECT pg_backend_pid()", &[]) + .await + .unwrap() + .get::<_, i32>(0); + + let busy = hold_migration_lock(&waiter, "main", 1).await.unwrap_err(); + + assert!(is_migration_lock_busy(&busy), "{busy:?}"); + assert!( + busy.to_string().contains(&format!("pid {holder_pid}:")), + "{busy}" + ); + } + #[test] fn validate_migration_name_accepts_safe_names() { for name in ["initial", "add_index_to_customers", "fix-bug_2", "ABC123"] { diff --git a/backend/windmill-api-workspaces/src/workspaces.rs b/backend/windmill-api-workspaces/src/workspaces.rs index 81fead981b..c94560314c 100644 --- a/backend/windmill-api-workspaces/src/workspaces.rs +++ b/backend/windmill-api-workspaces/src/workspaces.rs @@ -744,13 +744,15 @@ async fn is_premium( #[derive(Serialize)] struct BillableSeatsResponse { - /// Both omitted when the seats counted are another workspace's: a fork member need not be a + /// All three omitted when the seats counted are another workspace's: a fork member need not be a /// member of the billing root, so the root's headcount is not theirs to read. The total is, /// since it is the divisor of the quota their own executions draw on. #[serde(skip_serializing_if = "Option::is_none")] developers: Option, #[serde(skip_serializing_if = "Option::is_none")] operators: Option, + #[serde(skip_serializing_if = "Option::is_none")] + service_accounts: Option, seats: i64, } @@ -780,6 +782,7 @@ async fn get_billable_seats( Ok(Json(BillableSeatsResponse { developers: own.then_some(counted.developers), operators: own.then_some(counted.operators), + service_accounts: own.then_some(counted.service_accounts), seats: counted.seats, })) } @@ -7637,7 +7640,7 @@ async fn clone_scripts( dedicated_worker, ws_error_handler_muted, priority, timeout, delete_after_use, delete_after_secs, restart_unless_cancelled, concurrency_key, visible_to_runner_only, auto_kind, codebase, has_preprocessor, - on_behalf_of, on_behalf_of_email, assets, modules + on_behalf_of, on_behalf_of_email, assets, modules, job_token_scopes ) SELECT $1, hash, path, parent_hashes, summary, description, content, @@ -7669,7 +7672,7 @@ async fn clone_scripts( UNION ALL SELECT 1 FROM password p WHERE p.email = on_behalf_of AND p.super_admin)) - END, on_behalf_of_email, assets, modules + END, on_behalf_of_email, assets, modules, job_token_scopes FROM script WHERE workspace_id = $2"#, target_workspace_id, @@ -8027,7 +8030,8 @@ async fn clone_flows( workspace_id, path, summary, description, value, edited_by, edited_at, archived, schema, extra_perms, dependency_job, tag, ws_error_handler_muted, dedicated_worker, timeout, visible_to_runner_only, - concurrency_key, versions, on_behalf_of, on_behalf_of_email, lock_error_logs + concurrency_key, versions, on_behalf_of, on_behalf_of_email, lock_error_logs, + job_token_scopes ) SELECT $2, path, summary, description, value, edited_by, edited_at, archived, schema, extra_perms, NULL, tag, @@ -8053,7 +8057,7 @@ async fn clone_flows( UNION ALL SELECT 1 FROM password p WHERE p.email = on_behalf_of AND p.super_admin)) - END, on_behalf_of_email, lock_error_logs + END, on_behalf_of_email, lock_error_logs, job_token_scopes FROM flow WHERE workspace_id = $1", source_workspace_id, @@ -14046,6 +14050,12 @@ const TRIGGER_COMPARE_IGNORE: &[&str] = &[ "last_server_ping", "extra_perms", "permissioned_as", + // Schedule counters the scheduler bumps as occurrences are missed. A fork's + // schedules are disabled so theirs stay at the defaults, and no deploy carries + // them across: compared, a parent schedule that ever ran late differs forever. + "late_run_streak", + "missed_occurrences", + "last_missed_at", // Server-managed fields that the merge feature treats as workspace-local // (regenerated by the deploy handler): GCP `subscription_id` is rewritten // to `windmill__` in `CreateUpdate` mode, and Azure's diff --git a/backend/windmill-api-workspaces/src/workspaces_extra.rs b/backend/windmill-api-workspaces/src/workspaces_extra.rs index 8ca98f66d1..a0ff1f2370 100644 --- a/backend/windmill-api-workspaces/src/workspaces_extra.rs +++ b/backend/windmill-api-workspaces/src/workspaces_extra.rs @@ -408,8 +408,8 @@ pub(crate) async fn change_workspace_id( info!("Duplicating flow table rows"); sqlx::query!( "INSERT INTO flow - (workspace_id, path, summary, description, archived, extra_perms, dependency_job, tag, ws_error_handler_muted, dedicated_worker, timeout, visible_to_runner_only, on_behalf_of, on_behalf_of_email, concurrency_key, versions, value, schema, edited_by, edited_at, lock_error_logs) - SELECT $1, path, summary, description, archived, extra_perms, dependency_job, tag, ws_error_handler_muted, dedicated_worker, timeout, visible_to_runner_only, on_behalf_of, on_behalf_of_email, concurrency_key, versions, value, schema, edited_by, edited_at, lock_error_logs + (workspace_id, path, summary, description, archived, extra_perms, dependency_job, tag, ws_error_handler_muted, dedicated_worker, timeout, visible_to_runner_only, on_behalf_of, on_behalf_of_email, concurrency_key, versions, value, schema, edited_by, edited_at, lock_error_logs, job_token_scopes) + SELECT $1, path, summary, description, archived, extra_perms, dependency_job, tag, ws_error_handler_muted, dedicated_worker, timeout, visible_to_runner_only, on_behalf_of, on_behalf_of_email, concurrency_key, versions, value, schema, edited_by, edited_at, lock_error_logs, job_token_scopes FROM flow WHERE workspace_id = $2", &rw.new_id, &old_id @@ -1670,13 +1670,12 @@ pub async fn drop_forked_datatable_databases( match parent_pg.connect(Some(&db)).await { Ok((client, connection)) => { let join_handle = tokio::spawn(async move { connection.await }); - if let Err(e) = client - .execute(&format!("DROP DATABASE \"{}\"", db_to_drop), &[]) - .await - { + if let Err(e) = drop_database_with_sessions(&client, db_to_drop).await { errors.push(format!( "Could not drop database '{}' for datatable://{}: {}", - db_to_drop, dt_name, e + db_to_drop, + dt_name, + windmill_common::error::pg_error_message(&e) )); } drop(client); @@ -1695,6 +1694,44 @@ pub async fn drop_forked_datatable_databases( Ok(Json(errors)) } +/// Drops `dbname`, whose name the caller validated, along with the sessions still connected to +/// it: a worker keeps an idle connection to a database it ran a job on, and a plain +/// `DROP DATABASE` fails for as long as one exists. +async fn drop_database_with_sessions( + client: &tokio_postgres::Client, + dbname: &str, +) -> std::result::Result<(), tokio_postgres::Error> { + let version = client + .query_one("SELECT current_setting('server_version_num')::int", &[]) + .await? + .get::<_, i32>(0); + if version >= 130000 { + client + .execute(&format!("DROP DATABASE \"{dbname}\" WITH (FORCE)"), &[]) + .await?; + return Ok(()); + } + // No `FORCE` before 13. The drop below waits a few seconds for the sessions to exit, and + // reports the ones this role was not allowed to terminate. + if let Err(e) = client + .execute( + "SELECT pg_terminate_backend(pid) FROM pg_stat_activity + WHERE datname = $1 AND pid <> pg_backend_pid()", + &[&dbname], + ) + .await + { + tracing::warn!( + "Failed to terminate connections to '{dbname}': {}", + windmill_common::error::pg_error_message(&e) + ); + } + client + .execute(&format!("DROP DATABASE \"{dbname}\""), &[]) + .await?; + Ok(()) +} + /// Drop this fork workspace's ducklake namespaces: the `wm_fork_*` metadata schema in each /// lake's catalog database, plus (best effort) the fork's `__wm_forks//…` data files in /// the workspace storage. Driven by the `fork_ducklake_namespace` registry written at first @@ -2265,3 +2302,47 @@ async fn workspace_is_fork(db: &DB, w_id: &str) -> Result { .await? .unwrap_or(false)) } + +#[cfg(test)] +mod tests { + use super::drop_database_with_sessions; + + #[sqlx::test(migrations = false)] + async fn a_database_is_dropped_under_a_connected_session(pool: sqlx::PgPool) { + let mut config: tokio_postgres::Config = + std::env::var("DATABASE_URL").unwrap().parse().unwrap(); + let test_db = pool.connect_options().get_database().unwrap().to_string(); + let (client, connection) = config + .dbname(&test_db) + .connect(tokio_postgres::NoTls) + .await + .unwrap(); + tokio::spawn(connection); + // The test database's name already fills an identifier. + let held_db = format!("held_{}", &test_db[test_db.len().saturating_sub(40)..]); + client + .execute(&format!("CREATE DATABASE \"{held_db}\""), &[]) + .await + .unwrap(); + let (_session, connection) = config + .dbname(&held_db) + .connect(tokio_postgres::NoTls) + .await + .unwrap(); + tokio::spawn(connection); + + drop_database_with_sessions(&client, &held_db) + .await + .unwrap(); + + let left = client + .query_one( + "SELECT count(*) FROM pg_database WHERE datname = $1", + &[&held_db], + ) + .await + .unwrap() + .get::<_, i64>(0); + assert_eq!(left, 0); + } +} diff --git a/backend/windmill-api/openapi-deref.json b/backend/windmill-api/openapi-deref.json index 3cba2b7da6..8dae1bafcd 100644 --- a/backend/windmill-api/openapi-deref.json +++ b/backend/windmill-api/openapi-deref.json @@ -5424,6 +5424,10 @@ "is_past_due": { "type": "boolean" }, + "is_canceled": { + "type": "boolean", + "description": "the team plan was canceled this month; it stays premium until the month ends, capped at max_tolerated_executions" + }, "max_tolerated_executions": { "type": "number" } diff --git a/backend/windmill-api/openapi-deref.yaml b/backend/windmill-api/openapi-deref.yaml index d5a14a717d..439e4d11a0 100644 --- a/backend/windmill-api/openapi-deref.yaml +++ b/backend/windmill-api/openapi-deref.yaml @@ -5286,6 +5286,9 @@ paths: type: string is_past_due: type: boolean + is_canceled: + type: boolean + description: the team plan was canceled this month; it stays premium until the month ends, capped at max_tolerated_executions max_tolerated_executions: type: number required: diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index d9aeb63707..a81720308b 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -1,7 +1,7 @@ openapi: "3.0.3" info: - version: 1.821.0 + version: 1.825.0 title: Windmill API contact: @@ -4492,6 +4492,12 @@ paths: description: >- Omitted when the seats counted are another workspace's, as they are for a fork resolving to its billing root. + service_accounts: + type: integer + description: >- + Enabled service accounts, half a seat each like operators. Omitted when the + seats counted are another workspace's, as they are for a fork resolving to + its billing root. seats: type: integer required: @@ -4529,6 +4535,9 @@ paths: type: string is_past_due: type: boolean + is_canceled: + type: boolean + description: the team plan was canceled this month; it stays premium until the month ends, capped at max_tolerated_executions max_tolerated_executions: type: number required: @@ -16527,6 +16536,31 @@ paths: required: - database_length + /w/{workspace}/jobs/oldest: + get: + summary: get the creation date of the oldest job + operationId: getOldestJob + tags: + - job + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: all_workspaces + description: consider jobs from all workspaces (only valid if request come from the `admins` workspace) + in: query + schema: + type: boolean + responses: + "200": + description: creation date of the oldest job, absent when there is none + content: + application/json: + schema: + type: object + properties: + created_at: + type: string + format: date-time + /w/{workspace}/jobs/completed/count: get: summary: get completed count @@ -28478,6 +28512,8 @@ components: $ref: "../../openflow.openapi.yaml#/components/schemas/AiTransform" ProviderConfig: $ref: "../../openflow.openapi.yaml#/components/schemas/ProviderConfig" + ProviderTransform: + $ref: "../../openflow.openapi.yaml#/components/schemas/ProviderTransform" MemoryConfig: $ref: "../../openflow.openapi.yaml#/components/schemas/MemoryConfig" FlowModuleValue: @@ -28498,6 +28534,8 @@ components: $ref: "../../openflow.openapi.yaml#/components/schemas/BranchAll" AiAgent: $ref: "../../openflow.openapi.yaml#/components/schemas/AiAgent" + AiDecision: + $ref: "../../openflow.openapi.yaml#/components/schemas/AiDecision" Identity: $ref: "../../openflow.openapi.yaml#/components/schemas/Identity" FlowStatus: @@ -29462,6 +29500,8 @@ components: togetherai, aws_bedrock, customai, + typesafe, + cloudflare, ] GitSyncObjectType: @@ -30150,6 +30190,12 @@ components: items: type: string default: [] + job_token_scopes: + type: array + nullable: true + items: + type: string + description: "Scopes the token of every job of this runnable is restricted to (domain:action[:resource], e.g. oidc:write). Jobs it starts, flow steps and AI agent tools inherit the restriction. Unset, the job token carries the full permissions of the identity the job runs as; an empty list leaves it no API access beyond its own job." inherited_labels: type: array items: @@ -30307,6 +30353,12 @@ components: type: array items: type: string + job_token_scopes: + type: array + nullable: true + items: + type: string + description: "Scopes the token of every job of this runnable is restricted to (domain:action[:resource], e.g. oidc:write). Jobs it starts, flow steps and AI agent tools inherit the restriction. Unset, the job token carries the full permissions of the identity the job runs as; an empty list leaves it no API access beyond its own job. Omitting the field keeps the deployed value; null clears it." skip_draft_deletion: type: boolean description: "When true (set by the CLI / git sync), deploying this script does not delete an existing user draft at the same path." @@ -36165,6 +36217,12 @@ components: items: type: string default: [] + job_token_scopes: + type: array + nullable: true + items: + type: string + description: "Scopes the token of every job of this runnable is restricted to (domain:action[:resource], e.g. oidc:write). Jobs it starts, flow steps and AI agent tools inherit the restriction. Unset, the job token carries the full permissions of the identity the job runs as; an empty list leaves it no API access beyond its own job." inherited_labels: type: array items: @@ -36211,6 +36269,12 @@ components: type: array items: type: string + job_token_scopes: + type: array + nullable: true + items: + type: string + description: "Scopes the token of every job of this runnable is restricted to (domain:action[:resource], e.g. oidc:write). Jobs it starts, flow steps and AI agent tools inherit the restriction. Unset, the job token carries the full permissions of the identity the job runs as; an empty list leaves it no API access beyond its own job. Omitting the field keeps the deployed value; null clears it." required: - path # Like OpenFlowWPath but `path` is optional: on update the flow is identified by @@ -36247,6 +36311,12 @@ components: type: array items: type: string + job_token_scopes: + type: array + nullable: true + items: + type: string + description: "Scopes the token of every job of this runnable is restricted to (domain:action[:resource], e.g. oidc:write). Jobs it starts, flow steps and AI agent tools inherit the restriction. Unset, the job token carries the full permissions of the identity the job runs as; an empty list leaves it no API access beyond its own job. Omitting the field keeps the deployed value; null clears it." FlowPreview: type: object diff --git a/backend/windmill-api/src/agent_runs.rs b/backend/windmill-api/src/agent_runs.rs index d2e76dcca0..76f729e7e0 100644 --- a/backend/windmill-api/src/agent_runs.rs +++ b/backend/windmill-api/src/agent_runs.rs @@ -29,7 +29,7 @@ use crate::db::{ApiAuthed, DB}; use crate::jobs::{handle_chat_conversation_messages, set_flow_memory_id, RunJobQuery}; /// The id the agent's step carries in the run, as the editor's chat names it. A token scoped to -/// `jobs:run:agents:` reads its runs back by it (`require_job_within_run_scope`). +/// `jobs:run:agents:` reads its runs back by it (`require_job_within_read_scope`). const AGENT_NODE_ID: &str = "__wm_agent_root"; /// A turn of the agent's chat when `memory_id` names a conversation, a single run otherwise. @@ -84,6 +84,7 @@ pub(crate) async fn run_agent( )?; let push_authed = authed.clone().into(); + let scope_ceiling = windmill_api_auth::caller_scope_ceiling(&db, &authed).await?; let (uuid, mut tx) = push( &db, PushIsolationLevel::Isolated(user_db.clone(), authed.clone().into()), @@ -118,6 +119,7 @@ pub(crate) async fn run_agent( None, authed.trigger_or_fallback(None), None, + scope_ceiling.as_deref(), ) .await?; diff --git a/backend/windmill-api/src/ai.rs b/backend/windmill-api/src/ai.rs index 855cd29987..c401568868 100644 --- a/backend/windmill-api/src/ai.rs +++ b/backend/windmill-api/src/ai.rs @@ -246,6 +246,12 @@ struct AIStandardResource { /// Custom HTTP headers to include in AI requests #[serde(default)] headers: HashMap, + /// The API token of a `cloudflare` resource, which names it `token` rather than `api_key`. + #[serde(default, deserialize_with = "empty_string_as_none")] + token: Option, + /// The account of a `cloudflare` resource, which its Workers AI URL is built from. + #[serde(default, deserialize_with = "empty_string_as_none")] + account_id: Option, } #[derive(Deserialize, Debug)] @@ -293,10 +299,21 @@ async fn resolve_provider_credentials( // Skip get_base_url for Bedrock - it uses SDK directly, not HTTP let base_url = if matches!(provider, AIProvider::AWSBedrock) { String::new() + } else if *provider == AIProvider::Cloudflare && resource.base_url.is_none() { + let account_id = if let Some(account_id) = resource.account_id { + Some(resolve_var(account_id, db, w_id, user_db.as_ref(), authed).await?) + } else { + None + }; + windmill_ai::ai_providers::cloudflare_workers_ai_base_url(account_id.as_deref())? } else { provider.get_base_url(resource.base_url, db).await? }; - let api_key = if let Some(api_key) = resource.api_key { + let api_key = match provider { + AIProvider::Cloudflare => resource.api_key.or(resource.token), + _ => resource.api_key, + }; + let api_key = if let Some(api_key) = api_key { Some(resolve_var(api_key, db, w_id, user_db.as_ref(), authed).await?) } else { None @@ -486,9 +503,10 @@ async fn assume_bedrock_role( ) -> Result { #[cfg(all(feature = "enterprise", feature = "openidconnect", feature = "private"))] { - use windmill_common::oidc_oss::{generate_id_token, WorkspaceClaim}; + use windmill_common::oidc_oss::{generate_id_token, OidcCaller, WorkspaceClaim}; let id_token = generate_id_token( + OidcCaller::Bedrock, Some(db), WorkspaceClaim { workspace: w_id.to_string() }, windmill_ai::ai_bedrock::AWS_OIDC_AUDIENCE, @@ -1238,8 +1256,8 @@ async fn proxy( .get("X-Resource-Value") .map(|v| { let invalid = |e: String| Error::BadRequest(format!("Invalid X-Resource-Value: {e}")); - let decoded = - urlencoding::decode(v.to_str().unwrap_or("")).map_err(|e| invalid(e.to_string()))?; + let decoded = urlencoding::decode(v.to_str().unwrap_or("")) + .map_err(|e| invalid(e.to_string()))?; let value = serde_json::from_str::(&decoded) .map_err(|e| invalid(e.to_string()))?; check_scopes(&authed, || "resources:write".to_string())?; diff --git a/backend/windmill-api/src/ai_evals/run.rs b/backend/windmill-api/src/ai_evals/run.rs index 42c2fcb2f2..57e75b3f9f 100644 --- a/backend/windmill-api/src/ai_evals/run.rs +++ b/backend/windmill-api/src/ai_evals/run.rs @@ -760,6 +760,7 @@ async fn push_run_flow( let path = subject.path.clone(); let tx = PushIsolationLevel::Isolated(user_db.clone(), authed.clone().into()); + let scope_ceiling = windmill_api_auth::caller_scope_ceiling(db, authed).await?; let (uuid, tx) = push( db, tx, @@ -790,6 +791,7 @@ async fn push_run_flow( None, authed.trigger_or_fallback(None), None, + scope_ceiling.as_deref(), ) .await?; tx.commit().await?; diff --git a/backend/windmill-api/src/apps.rs b/backend/windmill-api/src/apps.rs index f01275c953..f276e6d143 100644 --- a/backend/windmill-api/src/apps.rs +++ b/backend/windmill-api/src/apps.rs @@ -764,6 +764,7 @@ async fn list_apps( path, value as "value!: sqlx::types::Json>", created_at, + email IS NULL as "legacy!", typ::text as "typ!" FROM draft WHERE workspace_id = $1 @@ -813,9 +814,9 @@ async fn list_apps( inherited_labels: None, is_draft: true, draft_path, - // Synthesized rows are the authed user's own draft. + // Owned by nobody when legacy; see scripts.rs. draft_users: Some(sqlx::types::Json(vec![DraftUserRef { - username: Some(authed.username.clone()), + username: (!row.legacy).then(|| authed.username.clone()), }])), }); } @@ -2755,6 +2756,7 @@ async fn create_app_internal<'a>( None, None, None, + None, ) .await?; tracing::info!("Pushed app dependency job {}", dependency_job_uuid); @@ -3896,6 +3898,7 @@ async fn update_app_internal<'a>( None, None, None, + None, ) .await?; tracing::info!("Pushed app dependency job {}", dependency_job_uuid); @@ -4490,6 +4493,10 @@ async fn execute_component( let app_trigger = (!is_preview).then(|| TriggerMetadata::new(Some(path.to_string()), JobTriggerKind::App)); + let scope_ceiling = match opt_authed.as_ref() { + Some(authed) => windmill_api_auth::caller_scope_ceiling(&db, authed).await?, + None => None, + }; let (uuid, mut tx) = push( &db, tx, @@ -4525,6 +4532,7 @@ async fn execute_component( end_user_email, app_trigger, None, + scope_ceiling.as_deref(), ) .await?; diff --git a/backend/windmill-api/src/apps_raw_bundle.rs b/backend/windmill-api/src/apps_raw_bundle.rs index f50f94f8eb..61dc4b861d 100644 --- a/backend/windmill-api/src/apps_raw_bundle.rs +++ b/backend/windmill-api/src/apps_raw_bundle.rs @@ -148,6 +148,7 @@ pub(crate) async fn bundle_raw_app_sources( args.insert("runnables".to_string(), runnables.to_owned()); let tx = PushIsolationLevel::Isolated(user_db.clone(), authed.clone().into()); + let scope_ceiling = windmill_api_auth::caller_scope_ceiling(db, authed).await?; let (uuid, tx) = push( db, tx, @@ -191,6 +192,7 @@ pub(crate) async fn bundle_raw_app_sources( None, authed.trigger_or_fallback(None), None, + scope_ceiling.as_deref(), ) .await?; tx.commit().await?; diff --git a/backend/windmill-api/src/concurrency_groups.rs b/backend/windmill-api/src/concurrency_groups.rs index c94608ac7e..aab73c81d9 100644 --- a/backend/windmill-api/src/concurrency_groups.rs +++ b/backend/windmill-api/src/concurrency_groups.rs @@ -56,6 +56,7 @@ async fn get_concurrency_key( .get_authed(Some(job.workspace_id.clone()), &token) .await .ok_or_else(not_found)?; + windmill_api_auth::check_job_token_scope(&authed_in_workspace, || "jobs:read".to_string())?; require_job_read_access( &db, diff --git a/backend/windmill-api/src/drafts.rs b/backend/windmill-api/src/drafts.rs index 7b2268e38f..a3cd34f3f8 100644 --- a/backend/windmill-api/src/drafts.rs +++ b/backend/windmill-api/src/drafts.rs @@ -884,9 +884,9 @@ async fn move_draft( {attempted}. Reopen it, re-save to rewrite it cleanly, then retry." ) } else if row.legacy { - // The home list synthesizes a draft-only row for the legacy draft with the - // caller's own name on it, so this is reachable from the row menu. Only an - // admin can claim or discard that row, and only from the drafts page. + // The home list surfaces the legacy draft as a draft-only row owned by nobody, + // so this is reachable from the row menu. It can be discarded (a write-gated + // legacy delete) or claimed by an admin, but never moved. format!( "'{path}' is a legacy workspace draft with no owner, so it cannot be moved. \ A workspace admin can claim or discard it on the Review & deploy drafts page." diff --git a/backend/windmill-api/src/jobs.rs b/backend/windmill-api/src/jobs.rs index 785c6ff935..1f92c6d425 100644 --- a/backend/windmill-api/src/jobs.rs +++ b/backend/windmill-api/src/jobs.rs @@ -19,7 +19,7 @@ use serde_json::value::RawValue; use serde_json::Value; use sha2::{Digest, Sha256}; use std::borrow::Cow; -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use std::str::FromStr; use std::sync::Arc; use std::time::Instant; @@ -292,6 +292,7 @@ pub fn workspaced_service() -> Router { ) .route("/run/dynamic_select", post(run_dynamic_select)) .route("/list", get(list_jobs)) + .route("/oldest", get(get_oldest_job)) .route("/asset_dispatch_edges", get(list_asset_dispatch_edges)) .route( "/list_selected_job_groups", @@ -497,6 +498,28 @@ async fn get_result_by_id( view_token.as_deref(), ) .await?; + // The node is resolved up through the enclosing flows, so a read path that admitted + // `flow_id` as a step must also admit the run it is a step of. A path-scoped + // `jobs:read` does not reach this route; a `jobs:run` scope beside it does. + if windmill_api_auth::scopes::job_read_confinement(authed.scopes.as_deref()) + .is_some_and(|c| c.has_read_paths()) + { + let top = sqlx::query_scalar::<_, Uuid>( + "WITH RECURSIVE chain(id, parent_job) AS ( + SELECT id, parent_job FROM v2_job WHERE id = $1 AND workspace_id = $2 + UNION ALL + SELECT j.id, j.parent_job FROM v2_job j + JOIN chain c ON j.id = c.parent_job AND j.workspace_id = $2 + ) + SELECT id FROM chain WHERE parent_job IS NULL", + ) + .bind(flow_id) + .bind(&w_id) + .fetch_optional(&db) + .await? + .ok_or_else(|| Error::NotFound(format!("Job {flow_id} not found")))?; + require_job_within_read_scope(&db, &authed, &w_id, &top).await?; + } let res = windmill_queue::get_result_by_id(db.clone(), w_id.clone(), flow_id, node_id, json_path) @@ -620,7 +643,7 @@ async fn cancel_job_api( // right to kill someone else's run. Anonymous callers are instead confined to // anonymous-created jobs by `cancel_job`'s `require_anonymous`. if let Some(authed) = opt_authed.as_ref() { - require_job_update_read_access(&db, &user_db, authed, &w_id, &id, None).await?; + require_job_cancel_access(&db, &user_db, authed, &w_id, &id).await?; } let tx = db.begin().await?; @@ -689,7 +712,15 @@ async fn cancel_persistent_script_api( Json(CancelJob { reason }): Json, ) -> error::Result<()> { let audit_author: AuditAuthor = match opt_authed { - Some(authed) => (&authed).into(), + Some(authed) => { + let path = script_path.to_path(); + if !windmill_api_auth::scopes::job_cancel_path_confinement(authed.scopes.as_deref()) + .is_none_or(|c| windmill_api_auth::scopes::cancel_confinement_admits(&c, path)) + { + return Err(Error::NotFound(format!("Script {path} not found"))); + } + (&authed).into() + } None => { return Err(Error::BadRequest(format!( "Cancelling persistent script require to be logged in and member of {w_id}" @@ -782,7 +813,7 @@ async fn force_cancel( // caller who can only see an inner step kill a root flow hidden from them. if let Some(authed) = opt_authed.as_ref() { let target = force_cancel_target(&db, &w_id, id).await?; - require_job_update_read_access(&db, &user_db, authed, &w_id, &target, None).await?; + require_job_cancel_access(&db, &user_db, authed, &w_id, &target).await?; } let tx = db.begin().await?; @@ -1705,6 +1736,22 @@ pub(crate) async fn require_job_read_access( job_id: &Uuid, created_by: &str, view_token: Option<&str>, +) -> error::Result<()> { + require_job_access( + db, user_db, authed, w_id, job_id, created_by, view_token, true, + ) + .await +} + +async fn require_job_access( + db: &DB, + user_db: &UserDB, + authed: &ApiAuthed, + w_id: &str, + job_id: &Uuid, + created_by: &str, + view_token: Option<&str>, + read_confined: bool, ) -> error::Result<()> { // Tag scope (`if_jobs:filter_tags:`) is an orthogonal hard restriction on a // scoped token: it must never read a job outside its allowed tags, regardless of @@ -1729,10 +1776,12 @@ pub(crate) async fn require_job_read_access( } } - // A path-scoped `jobs:run` token is likewise hard-restricted to the runnables it - // may start, ahead of every grant below — the token is handed out to run one thing, - // so it must not read jobs of anything else merely because its owner could. - require_job_within_run_scope(db, authed, w_id, job_id).await?; + // A path-scoped `jobs:run` or `jobs:read` token is likewise hard-restricted to the + // runnables it names, ahead of every grant below — the token is handed out for one + // thing, so it must not read jobs of anything else merely because its owner could. + if read_confined { + require_job_within_read_scope(db, authed, w_id, job_id).await?; + } // Fast path: you can always read a job you launched. This is also load-bearing // for apps — a component job runs as the app policy's `permissioned_as`, but its @@ -1880,20 +1929,35 @@ pub(crate) async fn require_job_read_access( /// could start an app's inline-script component but not read the run back — those jobs /// are `AppScript`/`Preview` kinds that no `jobs:run` scope can name. /// -/// No-op — and no query — for every caller whose job reads are not run-confined (see -/// `job_read_run_confinement`), which is all sessions, unscoped tokens and `jobs:read` -/// tokens. -async fn require_job_within_run_scope( +/// A `jobs:read:` scope confines the same way, by path alone: the job, or one of +/// its ancestors, must be a run of a deployed script, flow or agent at a named path. +/// Only those kinds count, because a preview's `runnable_path` is whatever its caller +/// sent and would otherwise let any preview pass for a named runnable. +/// +/// No-op — and no query — for every caller whose job reads are not confined (see +/// `job_read_confinement`), which is all sessions, unscoped tokens and tokens whose +/// `jobs:read` names no path. +async fn require_job_within_read_scope( db: &DB, authed: &ApiAuthed, w_id: &str, job_id: &Uuid, ) -> error::Result<()> { let Some(confinement) = - windmill_api_auth::scopes::job_read_run_confinement(authed.scopes.as_deref()) + windmill_api_auth::scopes::job_read_confinement(authed.scopes.as_deref()) else { return Ok(()); }; + // A job whose token is restricted by `job_token_scopes` still reads its own flow run + // (step results, user state), which the route check lets through whatever the + // scopes. A read path names what else it may read, not a fence around its own run. + if let Some(own_job) = authed.job_id { + if confinement.has_read_paths() + && windmill_api_auth::auth::job_in_same_flow_run(db, own_job, *job_id).await + { + return Ok(()); + } + } // `scope_kind` is the runnable kind a `jobs:run::` scope can name, or // NULL for a job no such scope reaches directly (previews, dependency jobs, // flow-inlined scripts) — those are still readable as a step of a matching flow, @@ -1941,13 +2005,7 @@ async fn require_job_within_run_scope( let in_scope = chain.iter().any( |job| match (job.runnable_path.as_deref(), job.scope_kind.as_deref()) { - (Some(runnable_path), Some(kind)) - if windmill_api_auth::scopes::run_confinement_admits( - &confinement, - kind, - runnable_path, - ) => - { + (Some(runnable_path), Some(kind)) if confinement.admits(kind, runnable_path) => { true } _ => job.launched_by_app.as_deref().is_some_and(&runs_app), @@ -1961,6 +2019,73 @@ async fn require_job_within_run_scope( } } +/// The `ids` a path-scoped `jobs:cancel:` token may cancel; all of them for +/// every caller whose cancels are not path-confined (see `job_cancel_path_confinement`). +/// +/// A job is admitted when it, or any of its `parent_job` ancestors, is a run of a +/// deployed script, flow or agent whose path the scope names: cancelling a flow's step +/// is within a scope on the flow. Only those kinds count, because a preview's +/// `runnable_path` is whatever its caller sent and would otherwise let any preview +/// impersonate an in-scope runnable. Agent runs are previews filed under the agent's +/// path, recognized the same way `require_job_within_read_scope` does. +async fn filter_jobs_within_cancel_scope( + db: &DB, + authed: &ApiAuthed, + w_id: &str, + ids: Vec, +) -> error::Result> { + let Some(confinement) = + windmill_api_auth::scopes::job_cancel_path_confinement(authed.scopes.as_deref()) + else { + return Ok(ids); + }; + let chain = sqlx::query!( + r#"WITH RECURSIVE chain(origin, id, parent_job) AS ( + SELECT id, id, parent_job FROM v2_job WHERE id = ANY($1) AND workspace_id = $2 + UNION ALL + SELECT c.origin, j.id, j.parent_job FROM v2_job j + JOIN chain c ON j.id = c.parent_job AND j.workspace_id = $2 + ) + SELECT c.origin AS "origin!", + CASE WHEN a.agent THEN regexp_replace(j.runnable_path, '\.chat$', '') + ELSE j.runnable_path END AS "runnable_path!" + FROM chain c JOIN v2_job j ON j.id = c.id, + LATERAL (SELECT j.kind = 'flowpreview' + AND j.raw_flow->'modules'->1 IS NULL + AND j.raw_flow->'modules'->0->>'id' = '__wm_agent_root' AS agent) a + WHERE j.workspace_id = $2 AND j.runnable_path IS NOT NULL + AND (a.agent OR j.kind IN ('script', 'script_hub', 'unassigned_script', 'flow', + 'unassigned_flow', 'singlestepflow', 'unassigned_singlestepflow'))"#, + &ids, + w_id, + ) + .fetch_all(db) + .await?; + let admitted: HashSet = chain + .into_iter() + .filter(|r| { + windmill_api_auth::scopes::cancel_confinement_admits(&confinement, &r.runnable_path) + }) + .map(|r| r.origin) + .collect(); + Ok(ids.into_iter().filter(|id| admitted.contains(id)).collect()) +} + +async fn require_job_within_cancel_scope( + db: &DB, + authed: &ApiAuthed, + w_id: &str, + job_id: Uuid, +) -> error::Result<()> { + if filter_jobs_within_cancel_scope(db, authed, w_id, vec![job_id]) + .await? + .is_empty() + { + return Err(Error::NotFound(format!("Job {job_id} not found"))); + } + Ok(()) +} + /// Self + every `parent_job` ancestor (intermediate sub-flows up to the top-level /// root) of `job_id`, resolved via the root DB (flow lineage is not sensitive). /// Falls back to `[job_id]` if the row is absent so callers still run their probe. @@ -2156,6 +2281,41 @@ async fn require_job_update_read_access( require_job_read_access(db, user_db, authed, w_id, job_id, &created_by, view_token).await } +/// The per-job check of the cancel routes. A `jobs:cancel` grant stands on its own, like +/// `jobs:write`: its paths confine it (`filter_jobs_within_cancel_scope`), not the +/// token's `jobs:run` scopes, which bound what a run token may read. Intersecting the two +/// would not hold anyway, since the token can mint itself a child holding only the +/// cancel scope. +async fn require_job_cancel_access( + db: &DB, + user_db: &UserDB, + authed: &ApiAuthed, + w_id: &str, + job_id: &Uuid, +) -> error::Result<()> { + let created_by = sqlx::query_scalar!( + "SELECT created_by FROM v2_job WHERE id = $1 AND workspace_id = $2", + job_id, + w_id, + ) + .fetch_optional(db) + .await? + .ok_or_else(|| Error::NotFound(format!("Job {job_id} not found")))?; + let read_confined = !windmill_api_auth::scopes::has_job_cancel_grant(authed.scopes.as_deref()); + require_job_access( + db, + user_db, + authed, + w_id, + job_id, + &created_by, + None, + read_confined, + ) + .await?; + require_job_within_cancel_scope(db, authed, w_id, *job_id).await +} + /// Whether a validated approval token should grant the job-read bypass. The token alone /// is sufficient unless the current approval step has `user_auth_required`, in which case /// only an authorized approver may read the job (and thus its args/flow inputs). @@ -2286,7 +2446,7 @@ async fn get_job( // an approval link for a job must not let a scoped token read one outside the // runnables it may start. if let Some(authed) = opt_authed.as_ref() { - require_job_within_run_scope(&db, authed, &w_id, &id).await?; + require_job_within_read_scope(&db, authed, &w_id, &id).await?; } } else { require_opt_authed_job_read_access( @@ -3204,6 +3364,7 @@ fn flow_tree_entry_label( "forloopflow" => " forloop", "whileloopflow" => " whileloop", "aiagent" => " ai-agent", + "aidecision" => " ai-decision", _ => "", }; @@ -4260,6 +4421,17 @@ struct ListableQueuedJob { pub workspace_id: String, } +/// Narrows a job list to what a path-scoped `jobs:read:` token may read: the +/// top-level runs of the runnables it names. A step is read by id, under its flow. No-op +/// for every caller whose lists are not path-filtered (see `job_list_path_filter`). +fn confine_list_to_read_scope(sqlb: &mut SqlBuilder, authed: &ApiAuthed) { + if let windmill_api_auth::ScopePathFilter::Restricted { exact, prefix } = + windmill_api_auth::scopes::job_list_path_filter(authed.scopes.as_deref()) + { + and_where_root_run_at_paths(sqlb, &exact, &prefix); + } +} + async fn list_queue_jobs( authed: ApiAuthed, Extension(user_db): Extension, @@ -4281,7 +4453,7 @@ async fn list_queue_jobs( "null as args" }; - let sql = list_queue_jobs_query( + let mut sqlb = list_queue_jobs_query( &w_id, &lq, &[ @@ -4311,8 +4483,9 @@ async fn list_queue_jobs( pagination, false, get_scope_tags(&authed), - ) - .sql()?; + ); + confine_list_to_read_scope(&mut sqlb, &authed); + let sql = sqlb.sql()?; let mut tx = user_db.begin(&authed).await?; let jobs = sqlx::query_as::<_, ListableQueuedJob>(&sql) .fetch_all(&mut *tx) @@ -4372,6 +4545,38 @@ async fn cancel_selection( let force_cancel = query.force_cancel.unwrap_or(false); let mut cancelled = Vec::new(); for (workspace_id, ids) in jobs_by_workspace { + let ids = + if windmill_api_auth::scopes::job_cancel_path_confinement(authed.scopes.as_deref()) + .is_some() + { + // Checked on the job the cancel actually kills: a force cancel reaches the + // highest queued ancestor. + let mut targets = Vec::with_capacity(ids.len()); + for id in ids { + let target = if force_cancel { + force_cancel_target(&db, &workspace_id, id).await? + } else { + id + }; + targets.push((id, target)); + } + let admitted: HashSet = filter_jobs_within_cancel_scope( + &db, + &authed, + &workspace_id, + targets.iter().map(|(_, t)| *t).collect(), + ) + .await? + .into_iter() + .collect(); + targets + .into_iter() + .filter(|(_, t)| admitted.contains(t)) + .map(|(id, _)| id) + .collect() + } else { + ids + }; let Json(mut w_cancelled) = cancel_jobs( ids, &db, @@ -4567,6 +4772,41 @@ async fn count_completed_jobs( )) } +#[derive(Deserialize)] +struct OldestJobQuery { + all_workspaces: Option, +} + +#[derive(Serialize)] +struct OldestJob { + #[serde(skip_serializing_if = "Option::is_none")] + created_at: Option>, +} + +/// Where the job history starts, whatever the caller may see of it: a client scanning the +/// runs backwards in time windows needs a bound that row-level visibility cannot make slow. +async fn get_oldest_job( + Extension(db): Extension, + Path(w_id): Path, + Query(q): Query, +) -> error::JsonResult { + let created_at = if w_id == "admins" && q.all_workspaces.unwrap_or(false) { + // One index probe per workspace: a plain min() over the table has no index to walk. + sqlx::query_scalar( + "SELECT min(o.created_at) FROM workspace w CROSS JOIN LATERAL \ + (SELECT min(created_at) AS created_at FROM v2_job WHERE workspace_id = w.id) o", + ) + .fetch_one(&db) + .await? + } else { + sqlx::query_scalar("SELECT min(created_at) FROM v2_job WHERE workspace_id = $1") + .bind(&w_id) + .fetch_one(&db) + .await? + }; + Ok(Json(OldestJob { created_at })) +} + lazy_static::lazy_static! { /// 0 keeps the connection-wide statement_timeout. static ref LIST_JOBS_STATEMENT_TIMEOUT_SECS: u64 = @@ -4666,7 +4906,7 @@ async fn list_jobs( } let sqlc = if lq.running.is_none() { - Some(list_completed_jobs_query( + let mut sqlc = list_completed_jobs_query( &w_id, Some(per_page), 0, @@ -4674,7 +4914,9 @@ async fn list_jobs( cj_fields_ref, true, get_scope_tags(&authed), - )) + ); + confine_list_to_read_scope(&mut sqlc, &authed); + Some(sqlc) } else { None }; @@ -4701,6 +4943,7 @@ async fn list_jobs( true, get_scope_tags(&authed), ); + confine_list_to_read_scope(&mut sqlq, &authed); if let Some(sqlc) = sqlc { format!("{} UNION ALL {}", &sqlq.subquery()?, &sqlc.subquery()?,) @@ -5124,6 +5367,11 @@ async fn get_approval_info( "Must be logged in or provide a valid approval token".to_string(), )); } + // Nothing below goes through `require_job_read_access`, and an approval token is no + // way around a scoped token's confinement. + if let Some(authed) = opt_authed.as_ref() { + require_job_within_read_scope(&db, authed, &w_id, &job_id).await?; + } // Fetch job info #[derive(sqlx::FromRow)] @@ -5356,8 +5604,17 @@ async fn get_approval_info( // Possession of view rights over this approval is sufficient to mint a // share-read-link token for the flow: it only grants read (no resume), and only to // an authenticated workspace member, so it never widens what the approver can do. - let hmac = generate_view_token(&w_id, row.id, VIEW_TOKEN_DOMAIN, &db).await?; - let view_token = Some(format!("{}.{hmac}", row.id)); + // Not for a token whose reads are confined: the link would outlive it and serve any + // member, which is why `job_view_token` is refused to it too. + let confined = opt_authed.as_ref().is_some_and(|authed| { + windmill_api_auth::scopes::job_read_confinement(authed.scopes.as_deref()).is_some() + }); + let view_token = if confined { + None + } else { + let hmac = generate_view_token(&w_id, row.id, VIEW_TOKEN_DOMAIN, &db).await?; + Some(format!("{}.{hmac}", row.id)) + }; Ok(Json(ApprovalInfo { flow_id: row.id, @@ -5819,7 +6076,7 @@ pub async fn get_suspended_job_flow( // resume secret must not let a scoped token read a flow it may not run. Anonymous // approvers are unaffected. if let Some(authed) = authed.as_ref() { - require_job_within_run_scope(&db, authed, &w_id, &flow_id).await?; + require_job_within_read_scope(&db, authed, &w_id, &flow_id).await?; } let flow = GetQuery::new() @@ -6003,7 +6260,7 @@ pub async fn get_flow_user_state( // Reachable by a `jobs:run` token (it is one of the by-id routes a run needs), so // apply the same run-scope confinement as the other single-job reads. RLS below // still governs which jobs the owner's identity can see at all. - require_job_within_run_scope(&db, &authed, &w_id, &job_id).await?; + require_job_within_read_scope(&db, &authed, &w_id, &job_id).await?; let mut tx = user_db.begin(&authed).await?; let r = sqlx::query_scalar!( r#" @@ -7193,6 +7450,7 @@ pub async fn restart_flow( let tx = PushIsolationLevel::Isolated(user_db, authed.clone().into()); + let scope_ceiling = windmill_api_auth::caller_scope_ceiling(&db, &authed).await?; let (uuid, tx) = push( &db, tx, @@ -7230,6 +7488,7 @@ pub async fn restart_flow( None, authed.trigger_or_fallback(None), run_query.suspended_mode, + scope_ceiling.as_deref(), ) .await?; tx.commit().await?; @@ -7367,7 +7626,11 @@ pub async fn run_workflow_as_code( extra.insert(ENTRYPOINT_OVERRIDE.to_string(), to_raw_value(&entrypoint)); let args = PushArgs { args: &task.args.unwrap_or_else(HashMap::new), extra: Some(extra) }; check_tag_available_for_workspace(&db, &w_id, &run_query.tag, &args, &authed).await?; - check_scopes(&authed, || format!("jobs:run"))?; + // A job running its own task is its runtime, not a new run: a restricted job token keeps + // it (`is_own_job_runtime_route`), and the task inherits that job's restriction at push. + if authed.job_id != Some(job_id) { + check_scopes(&authed, || format!("jobs:run"))?; + } // The task becomes a child of `job_id`, runs its code and writes into its flow status, so // only that job itself (the SDK's `task` wrapper, on its `WM_TOKEN`) or an admin may push it. if authed.job_id != Some(job_id) && !authed.is_admin { @@ -7512,6 +7775,7 @@ pub async fn run_workflow_as_code( ) }; + let scope_ceiling = windmill_api_auth::caller_scope_ceiling(&db, &authed).await?; let (uuid, mut tx) = push( &db, tx, @@ -7542,6 +7806,7 @@ pub async fn run_workflow_as_code( None, None, None, + scope_ceiling.as_deref(), ) .await?; @@ -7825,6 +8090,7 @@ pub async fn run_wait_result_job_by_path_get( ) }; + let scope_ceiling = windmill_api_auth::caller_scope_ceiling(&db, &authed).await?; let (uuid, tx) = push( &db, tx, @@ -7855,6 +8121,7 @@ pub async fn run_wait_result_job_by_path_get( None, authed.trigger_or_fallback(None), run_query.suspended_mode, + scope_ceiling.as_deref(), ) .await?; tx.commit().await?; @@ -7973,6 +8240,7 @@ pub async fn run_wait_result_script_by_path_internal( ) }; + let scope_ceiling = windmill_api_auth::caller_scope_ceiling(&db, &authed).await?; let (uuid, tx) = push( &db, tx, @@ -8003,6 +8271,7 @@ pub async fn run_wait_result_script_by_path_internal( None, authed.trigger_or_fallback(None), run_query.suspended_mode, + scope_ceiling.as_deref(), ) .await?; tx.commit().await?; @@ -8055,6 +8324,7 @@ pub async fn run_wait_result_script_by_hash( timeout, has_preprocessor, labels, + job_token_scopes, runnable_settings: ScriptRunnableSettingsInline { concurrency_settings, debouncing_settings }, .. @@ -8088,6 +8358,7 @@ pub async fn run_wait_result_script_by_hash( ) }; + let scope_ceiling = windmill_api_auth::caller_scope_ceiling(&db, &authed).await?; let (uuid, tx) = push( &db, tx, @@ -8105,6 +8376,7 @@ pub async fn run_wait_result_script_by_hash( apply_preprocessor: !run_query.skip_preprocessor.unwrap_or(false) && has_preprocessor.unwrap_or(false), labels, + job_token_scopes, }, push_args, authed.display_username(), @@ -8131,6 +8403,7 @@ pub async fn run_wait_result_script_by_hash( None, authed.trigger_or_fallback(None), run_query.suspended_mode, + scope_ceiling.as_deref(), ) .await?; tx.commit().await?; @@ -8650,6 +8923,7 @@ async fn run_preview_script( let push_args = PushArgs { extra, args: &preview_args }; check_tag_available_for_workspace(&db, &w_id, &tag, &push_args, &authed).await?; + let scope_ceiling = windmill_api_auth::caller_scope_ceiling(&db, &authed).await?; let (uuid, tx) = push( &db, tx, @@ -8700,6 +8974,7 @@ async fn run_preview_script( None, authed.trigger_or_fallback(None), None, + scope_ceiling.as_deref(), ) .await?; tx.commit().await?; @@ -8784,7 +9059,8 @@ async fn run_inline_script_by_path( token, db, w_id, - InlineScriptTarget::Path(script_path.to_path().to_string()), + // Resolved to a version by `run_inline_script_inner`. + InlineScriptTarget::Path { path: script_path.to_path().to_string(), hash: 0 }, body.args, Some(user_db), ) @@ -8847,6 +9123,42 @@ async fn run_inline_script_inner( args: Option>>, user_db: Option, ) -> error::Result { + // An inline run executes with the caller's own token, so it cannot honour a script's + // `job_token_scopes`: such a script only runs as a job. A path is resolved here, once, and + // the version checked is the version run. + let (target, restricted) = match target { + InlineScriptTarget::Path { path, .. } => { + let authed_ref = authed.to_authed_ref(); + let info = get_latest_deployed_hash_for_path( + user_db + .as_ref() + .map(|db| UserDbWithAuthed { db: db.clone(), authed: &authed_ref }), + db.clone(), + &w_id, + &path, + ) + .await?; + let restricted = info.job_token_scopes.is_some(); + ( + InlineScriptTarget::Path { path, hash: info.hash }, + restricted, + ) + } + InlineScriptTarget::Hash(hash) => { + let restricted = windmill_common::get_script_info_for_hash(None, &db, &w_id, hash) + .await? + .job_token_scopes + .is_some(); + (InlineScriptTarget::Hash(hash), restricted) + } + }; + if restricted { + return Err(Error::BadRequest( + "This script restricts its job token (job_token_scopes) and cannot be run inline; \ + run it as a job instead" + .to_string(), + )); + } let utils = get_worker_internal_server_inline_utils()?; let authed_owned: windmill_common::db::Authed = authed.clone().into(); let result = utils.run_inline_script.as_ref()(RunInlineScriptFnParams { @@ -9033,6 +9345,7 @@ async fn run_bundle_preview_script( // tracing::info!("is_tar 1: {is_tar}"); // hmap.insert("") + let scope_ceiling = windmill_api_auth::caller_scope_ceiling(&db, &authed).await?; let (uuid, ntx) = push( &db, ltx, @@ -9079,6 +9392,7 @@ async fn run_bundle_preview_script( None, authed.trigger_or_fallback(None), None, + scope_ceiling.as_deref(), ) .await?; job_id = Some(uuid); @@ -9198,6 +9512,7 @@ async fn push_dependencies_job( req.temp_script_refs .map(|v| hm.insert("temp_script_refs".to_owned(), to_raw_value(&v))); + let scope_ceiling = windmill_api_auth::caller_scope_ceiling(db, authed).await?; let (uuid, tx) = push( db, PushIsolationLevel::IsolatedRoot(db.clone()), @@ -9232,6 +9547,7 @@ async fn push_dependencies_job( None, None, None, + scope_ceiling.as_deref(), ) .await?; tx.commit().await?; @@ -9330,6 +9646,7 @@ async fn push_flow_dependencies_job( req.temp_script_refs .map(|v| args_map.insert("temp_script_refs".to_string(), to_raw_value(&v))); + let scope_ceiling = windmill_api_auth::caller_scope_ceiling(db, authed).await?; let (uuid, tx) = push( db, PushIsolationLevel::IsolatedRoot(db.clone()), @@ -9360,6 +9677,7 @@ async fn push_flow_dependencies_job( None, None, None, + scope_ceiling.as_deref(), ) .await?; tx.commit().await?; @@ -9414,6 +9732,7 @@ async fn add_batch_jobs( ) -> error::JsonResult> { require_super_admin(&db, &authed).await?; + let mut job_token_scopes: Option> = None; let ( hash, path, @@ -9445,11 +9764,13 @@ async fn add_batch_jobs( dedicated_worker, timeout, runnable_settings, + job_token_scopes: script_job_token_scopes, .. // TODO: consider on_behalf_of_email and created_by for batch jobs } = get_latest_deployed_hash_for_path(Some(db_authed), db.clone(), &w_id, &path) .await? .prefetch_cached(&db) .await?; + job_token_scopes = script_job_token_scopes; ( Some(script_hash), Some(path), @@ -9496,7 +9817,7 @@ async fn add_batch_jobs( } else if let Some(path) = batch_info.path { let mut tx = user_db.clone().begin(&authed).await?; let value_json = sqlx::query!( - "SELECT coalesce(flow_version_lite.value, flow_version.value) as \"value!: sqlx::types::Json>\" FROM flow + "SELECT coalesce(flow_version_lite.value, flow_version.value) as \"value!: sqlx::types::Json>\", flow.job_token_scopes FROM flow LEFT JOIN flow_version ON flow_version.id = flow.versions[array_upper(flow.versions, 1)] LEFT JOIN flow_version_lite @@ -9507,6 +9828,7 @@ async fn add_batch_jobs( .fetch_optional(&mut *tx) .await? .ok_or_else(|| Error::internal_err(format!("not found flow at path {:?}", path)))?; + job_token_scopes = value_json.job_token_scopes; let value = serde_json::from_str::(value_json.value.get()).map_err(|err| { Error::internal_err(format!( @@ -9519,6 +9841,11 @@ async fn add_batch_jobs( "Path is required if no value is not provided" ))? }; + if windmill_common::scopes::validate_flow_step_job_token_scopes(&value)? { + windmill_common::min_version::MIN_VERSION_SUPPORTS_JOB_TOKEN_SCOPES + .assert() + .await?; + } add_virtual_items_if_necessary(&mut value.modules); let flow_status = FlowStatus::new(&value); ( @@ -9562,7 +9889,8 @@ async fn add_batch_jobs( let language = language.unwrap_or(ScriptLang::Deno); let tag = if let Some(dedicated_worker) = dedicated_worker { - if dedicated_worker && path.is_some() { + // Same rule as `push`: a dedicated worker would run it with its own unscoped token. + if dedicated_worker && path.is_some() && job_token_scopes.is_none() { windmill_common::worker::dedicated_worker_tag(&w_id, &path.clone().unwrap()) } else { format!("{}", language.as_str()) @@ -9630,8 +9958,8 @@ async fn add_batch_jobs( .await?; sqlx::query!( - "INSERT INTO job_perms (job_id, email, username, is_admin, is_operator, folders, groups, workspace_id) - SELECT unnest($1::uuid[]), $2, $3, $4, $5, $6, $7, $8", + "INSERT INTO job_perms (job_id, email, username, is_admin, is_operator, folders, groups, workspace_id, job_token_scopes) + SELECT unnest($1::uuid[]), $2, $3, $4, $5, $6, $7, $8, $9", &uuids, authed.email, authed.username, @@ -9640,6 +9968,7 @@ async fn add_batch_jobs( &[], &[], w_id, + job_token_scopes.as_deref() as Option<&[String]>, ) .execute(&mut *tx) .await?; @@ -9690,6 +10019,13 @@ async fn run_preview_flow_job( // jobs:run scope so a narrowly-scoped token cannot escape its scope. See run_preview_script. check_scopes(&authed, || format!("jobs:run"))?; require_path_read_access_for_preview(&authed, &raw_flow.path)?; + // Step restrictions apply to a preview as they do to a deployed run, so they are checked + // the same way: an invalid entry or an older worker would leave a step unrestricted. + if windmill_common::scopes::validate_flow_step_job_token_scopes(&raw_flow.value)? { + windmill_common::min_version::MIN_VERSION_SUPPORTS_JOB_TOKEN_SCOPES + .assert() + .await?; + } // A builder must be able to test what it composes, but the submitted value is not the stored // one: without this the preview is a way to run inline code the write path refuses. if authed.is_operator { @@ -9732,6 +10068,7 @@ async fn run_preview_flow_job( check_tag_available_for_workspace(&db, &w_id, &tag, &PushArgs::from(&flow_args), &authed) .await?; + let scope_ceiling = windmill_api_auth::caller_scope_ceiling(&db, &authed).await?; let (uuid, mut tx) = push( &db, tx, @@ -9766,6 +10103,7 @@ async fn run_preview_flow_job( None, authed.trigger_or_fallback(None), None, + scope_ceiling.as_deref(), ) .await?; @@ -9989,6 +10327,7 @@ async fn run_dynamic_select( let scheduled_for = run_query.get_scheduled_for(&db).await?; let tx = PushIsolationLevel::Isolated(user_db.clone(), authed.clone().into()); + let scope_ceiling = windmill_api_auth::caller_scope_ceiling(&db, &authed).await?; let (uuid, tx) = push( &db, tx, @@ -10035,6 +10374,7 @@ async fn run_dynamic_select( None, authed.trigger_or_fallback(None), None, + scope_ceiling.as_deref(), ) .await?; tx.commit().await?; @@ -10110,6 +10450,7 @@ pub async fn run_job_by_hash_inner( delete_after_use, delete_after_secs, labels, + job_token_scopes, .. } = script_info; @@ -10142,6 +10483,7 @@ pub async fn run_job_by_hash_inner( ) }; + let scope_ceiling = windmill_api_auth::caller_scope_ceiling(&db, &authed).await?; let (uuid, tx) = push( &db, tx, @@ -10159,6 +10501,7 @@ pub async fn run_job_by_hash_inner( apply_preprocessor: !run_query.skip_preprocessor.unwrap_or(false) && has_preprocessor.unwrap_or(false), labels, + job_token_scopes, }, push_args, authed.display_username(), @@ -10185,6 +10528,7 @@ pub async fn run_job_by_hash_inner( None, authed.trigger_or_fallback(trigger), run_query.suspended_mode, + scope_ceiling.as_deref(), ) .await?; tx.commit().await?; @@ -11145,7 +11489,7 @@ async fn list_completed_jobs( "null as args" }; - let sql = list_completed_jobs_query( + let mut sqlb = list_completed_jobs_query( &w_id, Some(per_page), offset, @@ -11185,8 +11529,9 @@ async fn list_completed_jobs( ], false, get_scope_tags(&authed), - ) - .sql()?; + ); + confine_list_to_read_scope(&mut sqlb, &authed); + let sql = sqlb.sql()?; let mut tx = user_db.begin(&authed).await?; let jobs = sqlx::query_as::<_, ListableCompletedJob>(&sql) .fetch_all(&mut *tx) @@ -11365,7 +11710,7 @@ async fn get_completed_job_result( // confinement that gate carries — re-apply it, as `get_job` does for the // approval token. Anonymous approval access is untouched. if let Some(authed) = opt_authed.as_ref() { - require_job_within_run_scope(&db, authed, &w_id, &id).await?; + require_job_within_read_scope(&db, authed, &w_id, &id).await?; } } else { require_opt_authed_job_read_access( diff --git a/backend/windmill-api/src/mcp/core.rs b/backend/windmill-api/src/mcp/core.rs index 472c7bbec9..8f3962da13 100644 --- a/backend/windmill-api/src/mcp/core.rs +++ b/backend/windmill-api/src/mcp/core.rs @@ -372,7 +372,8 @@ impl McpBackend for WindmillBackend { token: &str, workspace_id: &str, ) -> BackendResult { - self.auth_cache + let authed = self + .auth_cache .get_authed(Some(workspace_id.to_string()), token) .await .ok_or_else(|| { @@ -383,7 +384,11 @@ impl McpBackend for WindmillBackend { ), None, ) - }) + })?; + // Job token scopes never include MCP scopes, so a restricted job token gets none. + windmill_api_auth::check_job_token_scope(&authed, || "mcp:all".to_string()) + .map_err(|e| ErrorData::invalid_params(e.to_string(), None))?; + Ok(authed) } async fn runnable_list_fingerprint(&self, workspace_id: &str) -> BackendResult { diff --git a/backend/windmill-api/src/offboarding.rs b/backend/windmill-api/src/offboarding.rs index 26eb06c8e8..b7f83dfd96 100644 --- a/backend/windmill-api/src/offboarding.rs +++ b/backend/windmill-api/src/offboarding.rs @@ -891,8 +891,8 @@ async fn offboard_user_from_workspace<'c>( let flows_reassigned = sqlx::query_scalar!( r#"WITH inserted AS ( INSERT INTO flow - (workspace_id, path, summary, description, archived, extra_perms, dependency_job, tag, ws_error_handler_muted, dedicated_worker, timeout, visible_to_runner_only, on_behalf_of, on_behalf_of_email, concurrency_key, versions, value, schema, edited_by, edited_at, labels, lock_error_logs) - SELECT workspace_id, REGEXP_REPLACE(path, 'u/' || $2 || '/(.*)', $1 || '/\1'), summary, description, archived, extra_perms, dependency_job, tag, ws_error_handler_muted, dedicated_worker, timeout, visible_to_runner_only, on_behalf_of, on_behalf_of_email, concurrency_key, versions, value, schema, edited_by, edited_at, labels, lock_error_logs + (workspace_id, path, summary, description, archived, extra_perms, dependency_job, tag, ws_error_handler_muted, dedicated_worker, timeout, visible_to_runner_only, on_behalf_of, on_behalf_of_email, concurrency_key, versions, value, schema, edited_by, edited_at, labels, lock_error_logs, job_token_scopes) + SELECT workspace_id, REGEXP_REPLACE(path, 'u/' || $2 || '/(.*)', $1 || '/\1'), summary, description, archived, extra_perms, dependency_job, tag, ws_error_handler_muted, dedicated_worker, timeout, visible_to_runner_only, on_behalf_of, on_behalf_of_email, concurrency_key, versions, value, schema, edited_by, edited_at, labels, lock_error_logs, job_token_scopes FROM flow WHERE path LIKE ('u/' || $2 || '/%') AND workspace_id = $3 RETURNING 1 diff --git a/backend/windmill-api/src/runnables.rs b/backend/windmill-api/src/runnables.rs index 2cee0efe33..846089630b 100644 --- a/backend/windmill-api/src/runnables.rs +++ b/backend/windmill-api/src/runnables.rs @@ -371,7 +371,8 @@ fn draft_branch_sql(kind: &str) -> String { "SELECT '{kind}' as kind, o.path, o.summary, o.workspace_id, '{{}}'::jsonb as extra_perms, \ false as starred, false as archived, \ true as is_draft, true as draft_only, o.draft_path, \ - json_build_array(json_build_object('username', $2::text)) as draft_users, \ + json_build_array(json_build_object('username', \ + CASE WHEN o.legacy THEN NULL ELSE $2::text END)) as draft_users, \ NULL::text[] as labels, NULL::text[] as inherited_labels, \ NULL::bool as ws_error_handler_muted, o.created_at as edited_at, \ NULL::bigint as hash, o.language, o.script_kind, o.auto_kind, \ @@ -380,6 +381,7 @@ fn draft_branch_sql(kind: &str) -> String { o.created_at as sort_time, lower(COALESCE(NULLIF(o.summary, ''), o.draft_path, o.path)) as sort_name, 0::bigint as tiebreak \ FROM ( \ SELECT DISTINCT ON (d.path) d.workspace_id, d.path, d.created_at, \ + d.email IS NULL as legacy, \ COALESCE(d.value->>'summary', '') as summary, \ NULLIF(NULLIF(d.value->>'{typed_path}', ''), d.path) as draft_path, \ {kind_cols} \ diff --git a/backend/windmill-api/src/static_assets.rs b/backend/windmill-api/src/static_assets.rs index 66dc4da9df..78c2b48f3b 100644 --- a/backend/windmill-api/src/static_assets.rs +++ b/backend/windmill-api/src/static_assets.rs @@ -100,6 +100,19 @@ fn serve_path(path: &str, original_path: &str, query: Option<&str>) -> Response< res = res.header("X-Robots-Tag", "noindex, nofollow"); } + // The raw-app preview shell evaluates whatever js is posted to it, so it is + // only ever served opaque-origin, with the deployed app wrapper's exact flags + // (`get_raw_app_data`). Matched on the embedded file, not the request path, + // so `//ui_builder/...` is covered. The editor loads a same-origin blob: copy. + if path == "ui_builder/app-preview.html" { + res = res.header( + header::CONTENT_SECURITY_POLICY, + "sandbox allow-scripts allow-forms allow-popups \ + allow-popups-to-escape-sandbox allow-downloads allow-modals \ + allow-top-navigation", + ); + } + // Add Content-Security-Policy header for static assets when policy is set if !CSP_POLICY.is_empty() { if let Ok(header_value) = HeaderValue::try_from(CSP_POLICY.as_str()) { diff --git a/backend/windmill-api/src/token.rs b/backend/windmill-api/src/token.rs index a1a9206196..b7fbe96404 100644 --- a/backend/windmill-api/src/token.rs +++ b/backend/windmill-api/src/token.rs @@ -193,13 +193,18 @@ lazy_static! { ScopeOption { value: "jobs:read".to_string(), label: "Read".to_string(), - requires_resource_path: false, + requires_resource_path: true, }, ScopeOption { value: "jobs:write".to_string(), label: "Write".to_string(), requires_resource_path: false, }, + ScopeOption { + value: "jobs:cancel".to_string(), + label: "Cancel".to_string(), + requires_resource_path: true, + }, ScopeOption { value: "jobs:run:scripts".to_string(), label: "Run scripts".to_string(), diff --git a/backend/windmill-api/src/users.rs b/backend/windmill-api/src/users.rs index a7f9aeda89..39a213c860 100644 --- a/backend/windmill-api/src/users.rs +++ b/backend/windmill-api/src/users.rs @@ -665,8 +665,8 @@ async fn update_username_in_workpsace<'c>( // ---- flows ---- sqlx::query!( r#"INSERT INTO flow - (workspace_id, path, summary, description, archived, extra_perms, dependency_job, tag, ws_error_handler_muted, dedicated_worker, timeout, visible_to_runner_only, on_behalf_of, on_behalf_of_email, concurrency_key, versions, value, schema, edited_by, edited_at) - SELECT workspace_id, REGEXP_REPLACE(path,'u/' || $2 || '/(.*)','u/' || $1 || '/\1'), summary, description, archived, extra_perms, dependency_job, tag, ws_error_handler_muted, dedicated_worker, timeout, visible_to_runner_only, on_behalf_of, on_behalf_of_email, concurrency_key, versions, value, schema, edited_by, edited_at + (workspace_id, path, summary, description, archived, extra_perms, dependency_job, tag, ws_error_handler_muted, dedicated_worker, timeout, visible_to_runner_only, on_behalf_of, on_behalf_of_email, concurrency_key, versions, value, schema, edited_by, edited_at, job_token_scopes) + SELECT workspace_id, REGEXP_REPLACE(path,'u/' || $2 || '/(.*)','u/' || $1 || '/\1'), summary, description, archived, extra_perms, dependency_job, tag, ws_error_handler_muted, dedicated_worker, timeout, visible_to_runner_only, on_behalf_of, on_behalf_of_email, concurrency_key, versions, value, schema, edited_by, edited_at, job_token_scopes FROM flow WHERE path LIKE ('u/' || $2 || '/%') AND workspace_id = $3"#, new_username, diff --git a/backend/windmill-api/src/workspaces_export.rs b/backend/windmill-api/src/workspaces_export.rs index ea79aaa572..3c3e965248 100644 --- a/backend/windmill-api/src/workspaces_export.rs +++ b/backend/windmill-api/src/workspaces_export.rs @@ -119,6 +119,8 @@ struct ScriptMetadata { pub debouncing_settings: DebouncingSettings, #[serde(skip_serializing_if = "Option::is_none")] pub labels: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub job_token_scopes: Option>, #[serde(skip_serializing_if = "is_empty_extra_perms")] pub extra_perms: serde_json::Value, } @@ -906,6 +908,7 @@ pub(crate) async fn tarball_workspace( .then_some(true), modules: script.modules, labels: script.labels, + job_token_scopes: script.job_token_scopes, // Same opt-in contract as flow/app: the tarball only surfaces // ACLs when `?preserve_extra_perms=true`. Passing `Null` lets the // `is_empty_extra_perms` skip-serializer drop the field entirely. @@ -966,7 +969,7 @@ pub(crate) async fn tarball_workspace( { let flows = sqlx::query_as::<_, Flow>( - "SELECT flow.workspace_id, flow.path, flow.summary, flow.description, flow.archived, flow.extra_perms, flow.dedicated_worker, flow.tag, flow.ws_error_handler_muted, flow.timeout, flow.visible_to_runner_only, flow.on_behalf_of, flow.labels, flow_version.schema, flow_version.value, flow_version.created_at as edited_at, flow_version.created_by as edited_by + "SELECT flow.workspace_id, flow.path, flow.summary, flow.description, flow.archived, flow.extra_perms, flow.dedicated_worker, flow.tag, flow.ws_error_handler_muted, flow.timeout, flow.visible_to_runner_only, flow.on_behalf_of, flow.labels, flow.job_token_scopes, flow_version.schema, flow_version.value, flow_version.created_at as edited_at, flow_version.created_by as edited_by FROM flow LEFT JOIN flow_version ON flow_version.id = flow.versions[array_upper(flow.versions, 1)] WHERE flow.workspace_id = $1 AND flow.archived = false", diff --git a/backend/windmill-common/Cargo.toml b/backend/windmill-common/Cargo.toml index 96fa7ceb63..4d43206ebc 100644 --- a/backend/windmill-common/Cargo.toml +++ b/backend/windmill-common/Cargo.toml @@ -22,7 +22,7 @@ smtp = ["dep:mail-send", "dep:rustls", "dep:tokio-rustls", "dep:rustls-native-ce scoped_cache = [] cloud = [] dev_override = [] -openidconnect = ["dep:openidconnect"] +openidconnect = ["dep:openidconnect", "dep:rsa"] python = ["dep:windmill-parser-py"] [lib] @@ -33,6 +33,7 @@ path = "src/lib.rs" tar.workspace = true hmac.workspace = true sha2.workspace = true +ryu-js.workspace = true sha1.workspace = true thiserror.workspace = true anyhow.workspace = true diff --git a/backend/windmill-common/src/auth.rs b/backend/windmill-common/src/auth.rs index 86b7f173ac..17310bdbbc 100644 --- a/backend/windmill-common/src/auth.rs +++ b/backend/windmill-common/src/auth.rs @@ -362,6 +362,10 @@ pub struct JobPerms { pub groups: Vec, pub folders: Vec, pub end_user_email: Option, + /// The job's effective scopes, stored on its `job_perms` row at push and minted into its + /// token. + #[serde(default)] + pub job_token_scopes: Option>, } impl From for Authed { @@ -654,7 +658,9 @@ pub async fn get_job_perms<'a, E: sqlx::PgExecutor<'a>>( ) -> sqlx::Result> { sqlx::query_as!( JobPerms, - "SELECT email, username, is_admin, is_operator, groups, folders, end_user_email FROM job_perms WHERE job_id = $1 AND workspace_id = $2", + "SELECT email, username, is_admin, is_operator, groups, folders, end_user_email, + job_token_scopes + FROM job_perms WHERE job_id = $1 AND workspace_id = $2", job_id, w_id ) @@ -745,17 +751,29 @@ pub async fn create_token_for_owner( } else { get_job_perms(db, job_id, w_id).await }; - let job_authed = match job_perms { - Ok(Some(jp)) => jp.into(), - _ => { + let (job_authed, job_token_scopes) = match job_perms { + Ok(Some(mut jp)) => { + let scopes = jp.job_token_scopes.take(); + (jp.into(), scopes) + } + // A failed read must not mint as if the job had no row: that would drop its restriction. + Err(e) => { + return Err(Error::internal_err(format!( + "Could not read permissions for job {job_id}: {e:#}" + ))) + } + // Push writes a job's `job_perms` row and its scopes in one statement, so a job with no + // row was never restricted. + Ok(None) => { tracing::warn!("Could not get permissions for job {job_id} from job_perms table, getting permissions directly..."); - fetch_authed_from_permissioned_as(owner, email, w_id, db) + let authed = fetch_authed_from_permissioned_as(owner, email, w_id, db) .await .map_err(|e| { Error::internal_err(format!( "Could not get permissions directly for job {job_id}: {e:#}" )) - })? + })?; + (authed, None) } }; @@ -766,7 +784,7 @@ pub async fn create_token_for_owner( Some(*job_id), Some(label.to_string()), audit_span, - None, + crate::scopes::job_token_jwt_scopes(job_token_scopes), ) .await } diff --git a/backend/windmill-common/src/client.rs b/backend/windmill-common/src/client.rs index 48f3e0205b..985d3bbbfc 100644 --- a/backend/windmill-common/src/client.rs +++ b/backend/windmill-common/src/client.rs @@ -307,12 +307,13 @@ impl AuthedClient { workspace_id: &str, file_key: &str, storage: Option, + max_bytes: Option, ) -> anyhow::Result { let mut query = vec![("file_key", file_key.to_string())]; if let Some(storage) = storage { query.push(("storage", storage)); } - let response = self + let mut response = self .force_client .as_ref() .unwrap_or(&HTTP_CLIENT) @@ -331,6 +332,39 @@ impl AuthedClient { .context("Failed to send download_s3_file request") .map_err(|e| anyhow::anyhow!(e.to_string()))?; + if let Some(max_bytes) = max_bytes { + let size_error = || { + anyhow::anyhow!( + "S3 file '{file_key}' exceeds maximum supported download size of {max_bytes} bytes" + ) + }; + if response + .content_length() + .is_some_and(|len| len > max_bytes as u64) + { + return Err(size_error()); + } + + let status = response.status(); + let mut bytes = Vec::new(); + // Content-Length may be absent (including for decoded compressed responses). + while let Some(chunk) = response + .chunk() + .await + .context("Failed to read response bytes")? + { + if chunk.len() > max_bytes - bytes.len() { + return Err(size_error()); + } + bytes.extend_from_slice(&chunk); + } + return if status == reqwest::StatusCode::OK { + Ok(bytes.into()) + } else { + Err(anyhow::anyhow!(String::from_utf8_lossy(&bytes).into_owned())) + }; + } + match response.status().as_u16() { 200u16 => Ok(response .bytes() @@ -341,6 +375,78 @@ impl AuthedClient { } } +#[cfg(test)] +mod tests { + use super::*; + use std::time::Duration; + use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader}; + + async fn download_response( + response: &'static str, + max_bytes: Option, + ) -> anyhow::Result { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let client = AuthedClient::new( + format!("http://{}", listener.local_addr().unwrap()), + "test_ws".to_string(), + "test_token".to_string(), + Some(reqwest::Client::builder().no_proxy().build().unwrap()), + ); + let server = tokio::spawn(async move { + let (socket, _) = listener.accept().await.unwrap(); + let mut socket = BufReader::new(socket); + loop { + let mut line = String::new(); + assert!(socket.read_line(&mut line).await.unwrap() > 0); + if line == "\r\n" { + break; + } + } + socket + .get_mut() + .write_all(response.as_bytes()) + .await + .unwrap(); + // Keep incomplete bodies open so rejection must happen before EOF. + std::future::pending::<()>().await; + }); + let result = tokio::time::timeout( + Duration::from_secs(5), + client.download_s3_file("test_ws", "image.png", None, max_bytes), + ) + .await; + server.abort(); + result.expect("download waited for the oversized body to finish") + } + + #[tokio::test] + async fn download_s3_file_rejects_oversized_body_before_eof() { + for response in [ + "HTTP/1.1 200 OK\r\nContent-Length: 5\r\n\r\n", + "HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n4\r\nabcd\r\n1\r\ne\r\n", + "HTTP/1.1 500 Internal Server Error\r\nTransfer-Encoding: chunked\r\n\r\n5\r\nerror\r\n", + ] { + let error = download_response(response, Some(4)).await.unwrap_err(); + assert!(error.to_string().contains( + "S3 file 'image.png' exceeds maximum supported download size of 4 bytes" + )); + } + } + + #[tokio::test] + async fn download_s3_file_accepts_body_at_limit() { + for response in [ + "HTTP/1.1 200 OK\r\nContent-Length: 4\r\n\r\nabcd", + "HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n4\r\nabcd\r\n0\r\n\r\n", + ] { + assert_eq!( + download_response(response, Some(4)).await.unwrap(), + &b"abcd"[..] + ); + } + } +} + #[inline] fn query_from_json_path(json_path: Option) -> Vec<(&'static str, String)> { json_path diff --git a/backend/windmill-common/src/flow_conversations.rs b/backend/windmill-common/src/flow_conversations.rs index 5ee99d0681..bde4d5ccce 100644 --- a/backend/windmill-common/src/flow_conversations.rs +++ b/backend/windmill-common/src/flow_conversations.rs @@ -278,6 +278,8 @@ pub async fn add_message_to_conversation_tx( tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, conversation_id: Uuid, job_id: Option, + // The flow run the user message started; retention deletes the turn's rows by it. + turn_job_id: Uuid, content: &str, message_type: MessageType, step_name: Option<&str>, @@ -303,12 +305,13 @@ pub async fn add_message_to_conversation_tx( // Insert the message sqlx::query!( - "INSERT INTO flow_conversation_message (conversation_id, message_type, content, job_id, step_name, success, tool_arguments, tool_result, reasoning, attachments) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)", + "INSERT INTO flow_conversation_message (conversation_id, message_type, content, job_id, turn_job_id, step_name, success, tool_arguments, tool_result, reasoning, attachments) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11)", conversation_id, message_type as MessageType, content, job_id, + turn_job_id, step_name, success, extras.and_then(|e| e.tool_arguments.as_deref()), diff --git a/backend/windmill-common/src/flows.rs b/backend/windmill-common/src/flows.rs index b392dda89a..222694aeac 100644 --- a/backend/windmill-common/src/flows.rs +++ b/backend/windmill-common/src/flows.rs @@ -357,6 +357,8 @@ fn check_module_value_is_composition_only( } } } + // Runs no code: it sends its inputs to the decision model. + FlowModuleValue::AIDecision { tag, .. } => push_tag(tag), } Ok(()) } diff --git a/backend/windmill-common/src/global_settings.rs b/backend/windmill-common/src/global_settings.rs index 9137dc364e..e1a731f3e6 100644 --- a/backend/windmill-common/src/global_settings.rs +++ b/backend/windmill-common/src/global_settings.rs @@ -9,6 +9,7 @@ pub const DEFAULT_TAGS_WORKSPACES_SETTING: &str = "default_tags_workspaces"; pub const FORK_WORKSPACE_TAG_APPEND_FORK_SUFFIX_SETTING: &str = "fork_workspace_tag_append_fork_suffix"; pub const PREVIEW_TAGS_OVERRIDE_SETTING: &str = "preview_tags_override"; +pub const DEPENDENCY_JOB_TAG_SETTING: &str = "dependency_job_tag"; pub const BASE_URL_SETTING: &str = "base_url"; pub const WS_BASE_URL_SETTING: &str = "ws_base_url"; pub const OAUTH_SETTING: &str = "oauths"; @@ -430,8 +431,9 @@ pub const CANCEL_STRANDED_JOBS_AFTER_DAYS_SETTING: &str = "cancel_stranded_jobs_ /// server keeps to itself, add it here. pub const AGENT_WORKER_BLOCKED_SETTINGS: &[&str] = &[ // Instance identity / auth secrets — disclosure enables privilege escalation - // or impersonation. + // or impersonation. `rsa_keys` signs job OIDC tokens; only the server mints them. JWT_SECRET_SETTING, + "rsa_keys", OAUTH_SETTING, SMTP_SETTING, SCIM_TOKEN_SETTING, @@ -1143,6 +1145,7 @@ mod tests { // secrets. They must never be served by the agent-worker endpoint. for key in [ JWT_SECRET_SETTING, + "rsa_keys", OAUTH_SETTING, SMTP_SETTING, SCIM_TOKEN_SETTING, diff --git a/backend/windmill-common/src/instance_config.rs b/backend/windmill-common/src/instance_config.rs index fd790547b6..a4164cd2c5 100644 --- a/backend/windmill-common/src/instance_config.rs +++ b/backend/windmill-common/src/instance_config.rs @@ -275,6 +275,8 @@ pub struct GlobalSettings { #[serde(skip_serializing_if = "Option::is_none")] pub auto_build_binary_tag: Option, #[serde(skip_serializing_if = "Option::is_none")] + pub dependency_job_tag: Option, + #[serde(skip_serializing_if = "Option::is_none")] pub ws_base_url: Option, #[serde(skip_serializing_if = "Option::is_none")] pub github_app_webhook_base_url: Option, @@ -1019,10 +1021,40 @@ pub const PROTECTED_SETTINGS: &[&str] = &[ "min_keep_alive_version", ]; +/// Secrets the server signs with or generated for itself. `jwt_secret` signs API and job +/// tokens and `rsa_keys` signs job OIDC tokens, so reading either is enough to mint tokens +/// for any user. Superadmin reads return them only while `server_secrets_exported()`. +/// Withholding them from reads does not make them unwritable: the two signing keys are not +/// in `HIDDEN_SETTINGS`, so config can still set them (an operator ConfigMap may set +/// `jwt_secret`). +pub const SERVER_SECRET_SETTINGS: &[&str] = &[ + "jwt_secret", + "rsa_keys", + "custom_instance_replication_pwd", + "external_instance_pg_state", +]; + +/// Whether superadmin API reads (settings list, single-setting read, config export) return +/// `SERVER_SECRET_SETTINGS`. On by default so `wmill instance get-config` and `pull` carry a +/// full migration; `EXPORT_SERVER_SECRETS=false` withholds them from every API response, so a +/// leaked superadmin token cannot take the signing keys. Fails closed: once set to a +/// non-empty value, only `true`/`1`/`yes`/`on` keep exporting, so `0` or a typo withholds. +/// Read per call: the flag is cheap and tests flip it in-process. +pub fn server_secrets_exported() -> bool { + match std::env::var("EXPORT_SERVER_SECRETS") { + Ok(v) if !v.trim().is_empty() => matches!( + v.trim().to_ascii_lowercase().as_str(), + "true" | "1" | "yes" | "on" + ), + _ => true, + } +} + +pub fn is_withheld_server_secret(name: &str) -> bool { + SERVER_SECRET_SETTINGS.contains(&name) && !server_secrets_exported() +} + /// Internal settings that are never exposed via the API or included in config exports. -/// Note: jwt_secret is intentionally NOT hidden — it is included in YAML exports so that -/// operators can set it via ConfigMap. It is protected from deletion (PROTECTED_SETTINGS) -/// and from being set to empty/null, and its value is partially redacted in log output. pub const HIDDEN_SETTINGS: &[&str] = &[ "uid", "min_keep_alive_version", @@ -1567,6 +1599,21 @@ pub fn resolve_env_refs(settings: &mut GlobalSettings) -> Result<(), String> { impl InstanceConfig { /// Read the full instance configuration from the database. pub async fn from_db(db: &sqlx::Pool) -> anyhow::Result { + Self::read_from_db(db, true).await + } + + /// The instance configuration as an API response may carry it: `from_db` without + /// `SERVER_SECRET_SETTINGS` unless `server_secrets_exported()`. It holds admin-entered + /// credentials (`license_key`, `scim_token`, SMTP and OAuth secrets), so callers must + /// have checked superadmin. + pub async fn from_db_for_api(db: &sqlx::Pool) -> anyhow::Result { + Self::read_from_db(db, server_secrets_exported()).await + } + + async fn read_from_db( + db: &sqlx::Pool, + include_server_secrets: bool, + ) -> anyhow::Result { // Read global_settings table → flat map → deserialize into GlobalSettings let rows: Vec<(String, serde_json::Value)> = sqlx::query_as("SELECT name, value FROM global_settings") @@ -1575,6 +1622,9 @@ impl InstanceConfig { let map: serde_json::Map = rows .into_iter() .filter(|(name, _)| !HIDDEN_SETTINGS.contains(&name.as_str())) + .filter(|(name, _)| { + include_server_secrets || !SERVER_SECRET_SETTINGS.contains(&name.as_str()) + }) .collect(); let global_settings: GlobalSettings = serde_json::from_value(serde_json::Value::Object(map))?; diff --git a/backend/windmill-common/src/item_digest.rs b/backend/windmill-common/src/item_digest.rs new file mode 100644 index 0000000000..23110bcae7 --- /dev/null +++ b/backend/windmill-common/src/item_digest.rs @@ -0,0 +1,132 @@ +//! Content digests of deployed scripts and flows, reproducible with `wmill digest` +//! (`cli/src/commands/digest/digest.ts`) from a `wmill sync` checkout pulled after the +//! deployment's dependency jobs: they write the locks, and store loops and branches with +//! their defaults filled in. The two implementations must hash the same bytes: change one +//! only together with the other. Numbers hash as the doubles they parse to, which needs +//! serde_json's `float_roundtrip` to parse them as exactly as JavaScript does. +//! +//! A digest is the lowercase hex SHA-256 of the canonical JSON (RFC 8785) of the item, with +//! every object member whose value is null removed: +//! - a script: `{"content", "lock", "modules", "codebase"}`, where `modules` is the +//! multi-file script's `{: {"content", "lock"}}` and is left out when empty. +//! The language is not covered: a checkout only records it in the file extension; +//! - a flow step's inline script: `{"content", "lock"}`; +//! - a flow: its `value`. A flow whose value references code in `flow_node` has none, see +//! [`references_flow_nodes`]. + +use serde_json::Value; +use sha2::{Digest, Sha256}; + +use crate::flows::{Branch, FlowModule, FlowModuleValue, FlowValue, ToolValue}; + +pub fn digest(value: &Value) -> String { + let mut out = String::new(); + write_canonical(value, &mut out); + format!("{:x}", Sha256::digest(out.as_bytes())) +} + +pub fn canonical_json(value: &Value) -> String { + let mut out = String::new(); + write_canonical(value, &mut out); + out +} + +fn write_canonical(value: &Value, out: &mut String) { + match value { + Value::Null => out.push_str("null"), + Value::Bool(b) => out.push_str(if *b { "true" } else { "false" }), + // RFC 8785 serializes every number as an IEEE double, the way ECMAScript prints it. + Value::Number(n) => { + let f = n.as_f64().unwrap_or(0.0); + out.push_str(ryu_js::Buffer::new().format(f)) + } + Value::String(s) => out.push_str(&serde_json::to_string(s).unwrap_or_default()), + Value::Array(items) => { + out.push('['); + for (i, item) in items.iter().enumerate() { + if i > 0 { + out.push(','); + } + write_canonical(item, out); + } + out.push(']'); + } + Value::Object(members) => { + // RFC 8785 orders members by the UTF-16 code units of their names. + let mut members: Vec<_> = members.iter().filter(|(_, v)| !v.is_null()).collect(); + members.sort_by(|(a, _), (b, _)| a.encode_utf16().cmp(b.encode_utf16())); + out.push('{'); + for (i, (k, v)) in members.into_iter().enumerate() { + if i > 0 { + out.push(','); + } + out.push_str(&serde_json::to_string(k).unwrap_or_default()); + out.push(':'); + write_canonical(v, out); + } + out.push('}'); + } + } +} + +/// Whether a flow value runs code stored outside it, in `flow_node` rows: an inline script +/// step `{"type": "flowscript", "id"}`, or a loop's or branch's steps in a `*_node`. Its digest +/// would not cover that code, and a checkout cannot reproduce it either, since the export +/// carries the references. A value or step that does not parse runs nothing, so it counts as +/// none. +pub fn references_flow_nodes(value: &Value) -> bool { + let Ok(flow) = serde_json::from_value::(value.clone()) else { + return false; + }; + flow.modules + .iter() + .chain(flow.preprocessor_module.as_deref()) + .chain(flow.failure_module.as_deref()) + .any(module_references_flow_nodes) +} + +fn module_references_flow_nodes(module: &FlowModule) -> bool { + module + .get_value() + .is_ok_and(|v| module_value_references_flow_nodes(&v)) +} + +fn module_value_references_flow_nodes(value: &FlowModuleValue) -> bool { + let any = |modules: &[FlowModule]| modules.iter().any(module_references_flow_nodes); + let branches = |branches: &[Branch]| { + branches + .iter() + .any(|b| b.modules_node.is_some() || any(&b.modules)) + }; + match value { + FlowModuleValue::FlowScript { .. } => true, + FlowModuleValue::ForloopFlow { modules, modules_node, .. } + | FlowModuleValue::WhileloopFlow { modules, modules_node, .. } => { + modules_node.is_some() || any(modules) + } + FlowModuleValue::BranchOne { branches: b, default, default_node } => { + default_node.is_some() || any(default) || branches(b) + } + FlowModuleValue::BranchAll { branches: b, .. } => branches(b), + FlowModuleValue::AIAgent { tools, .. } => tools.iter().any(|tool| match &tool.value { + ToolValue::FlowModule(v) => module_value_references_flow_nodes(v), + _ => false, + }), + _ => false, + } +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + #[test] + fn canonical_form_follows_rfc_8785_without_nulls() { + let v = json!({"b": [1.0, 1e21, 0.1, -0.0, null], "a": null, "é": "\u{1f}\"\n", "\u{e000}": 1, "\u{1f600}": 2}); + assert_eq!( + canonical_json(&v), + "{\"b\":[1,1e+21,0.1,0,null],\"é\":\"\\u001f\\\"\\n\",\"\u{1f600}\":2,\"\u{e000}\":1}" + ); + } +} diff --git a/backend/windmill-common/src/job_provenance.rs b/backend/windmill-common/src/job_provenance.rs index 63075b0c37..96db3b8a87 100644 --- a/backend/windmill-common/src/job_provenance.rs +++ b/backend/windmill-common/src/job_provenance.rs @@ -1,6 +1,7 @@ +use serde_json::Value; use uuid::Uuid; -use crate::{db::DB, error::Result, jobs::JobKind, scripts::ScriptHash}; +use crate::{db::DB, error::Result, item_digest, jobs::JobKind, scripts::ScriptHash}; /// Where a job's code comes from, as far as its chain of parents can prove it. pub struct JobProvenance { @@ -28,7 +29,39 @@ pub struct JobProvenance { pub root_path: Option, pub root_kind: JobKind, pub root_trigger_kind: Option, + /// What triggered the root job, as recorded in `v2_job.trigger`. + pub root_trigger: Option, pub root_version: Option, + /// See [`item_digest`]. Only scripts, flows and flow steps' inline scripts have one. + pub digest: Option, + pub root_digest: Option, + /// The job's worker tag. + pub tag: String, + /// The group of the worker running the job, if it is running. + pub worker_group: Option, + pub run_as_type: RunAsType, +} + +/// What `permissioned_as` names. +#[derive(Debug, PartialEq, Eq, Clone, Copy)] +pub enum RunAsType { + User, + ServiceAccount, + Group, + /// No member of the workspace: a superadmin outside it, a built-in identity + /// (`superadmin_secret@windmill.dev`, ...) or a user since removed. + NonMember, +} + +impl RunAsType { + pub fn as_str(&self) -> &'static str { + match self { + RunAsType::User => "user", + RunAsType::ServiceAccount => "service_account", + RunAsType::Group => "group", + RunAsType::NonMember => "non_member", + } + } } #[derive(Debug, PartialEq, Eq)] @@ -58,6 +91,8 @@ struct LineageJob { runnable_path: Option, runnable_id: Option, trigger_kind: Option, + trigger: Option, + tag: String, permissioned_as: String, origin_verified: bool, current_version: bool, @@ -66,6 +101,10 @@ struct LineageJob { restarted_from: Option, app_stamped: bool, args_modules: bool, + worker_group: Option, + run_as_type: Option, + /// What the digest hashes, for the job and the root only. + digest_source: Option, } /// Reads any job of `w_id` regardless of the caller: authorize access to `job_id` first. @@ -74,16 +113,17 @@ pub async fn job_provenance(db: &DB, job_id: &Uuid, w_id: &str) -> Result