diff --git a/.agents/skills/pr/SKILL.md b/.agents/skills/pr/SKILL.md index 6a72949c96..f2a9ec8dad 100644 --- a/.agents/skills/pr/SKILL.md +++ b/.agents/skills/pr/SKILL.md @@ -210,7 +210,7 @@ the diff first — `git diff --name-only main...HEAD` answers most of these. **Flip without asking** when it is self-contained: a single-file fix, test-only, docs-only, one call site, no new public surface. -Unattended (webmux oneshot) there is nobody to ask, so the judgement holds and the action +Unattended (a one-shot run) there is nobody to ask, so the judgement holds and the action degrades: flip the self-contained ones, and leave the rest at a clean draft with a line in the PR description saying why — `left in draft: adds a migration, wants a human look before ready`. Don't flip a wide-blast-radius change just because the round came back clean, and don't ask a diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 80970d1e76..0d174a548f 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -1,4 +1,13 @@ -# Basic set up for three package managers +# Weekly, grouped version updates for every shipped manifest, plus one grouped +# security-update PR per manifest. Security-update PRs have no open-PR limit, so +# without an `applies-to: security-updates` group each alert would get its own PR. +# Bun has no security updates: /cli alerts are fixed through its version group. +# +# Deliberately NOT covered here: +# - /rust-client: Cargo.toml is generated and gitignored +# - /backend/parsers/windmill-parser-wasm: excluded from the workspace, needs nightly cargo +# - /benchmarks/pulumi: one-off load-test infrastructure, not shipped +# - /integration_tests: CI-only version: 2 updates: @@ -13,12 +22,32 @@ updates: directory: "/frontend" schedule: interval: "weekly" + groups: + # minor + patch bumps in one PR; majors in their own PR so they can be + # reviewed separately without blocking the routine bumps + frontend-deps: + patterns: ["*"] + update-types: ["minor", "patch"] + frontend-majors: + patterns: ["*"] + update-types: ["major"] + frontend-security: + applies-to: security-updates + patterns: ["*"] # Maintain dependencies for cargo - package-ecosystem: "cargo" directory: "/backend" schedule: interval: "weekly" + groups: + # majors stay as individual PRs: they usually need code changes + cargo-deps: + patterns: ["*"] + update-types: ["minor", "patch"] + cargo-security: + applies-to: security-updates + patterns: ["*"] # Maintain dependencies for Docker - package-ecosystem: "docker" @@ -31,3 +60,52 @@ updates: directory: "/python-client/wmill" schedule: interval: "weekly" + groups: + python-client-security: + applies-to: security-updates + patterns: ["*"] + + # Maintain dependencies for the CLI (bun.lock) + - package-ecosystem: "bun" + directory: "/cli" + schedule: + interval: "weekly" + groups: + cli-deps: + patterns: ["*"] + + # Maintain dependencies for the YAML validator + - package-ecosystem: "npm" + directory: "/windmill-yaml-validator" + schedule: + interval: "weekly" + groups: + yaml-validator-deps: + patterns: ["*"] + yaml-validator-security: + applies-to: security-updates + patterns: ["*"] + + # Maintain dependencies for the TypeScript client (devDependencies only) + - package-ecosystem: "npm" + directory: "/typescript-client" + schedule: + interval: "weekly" + groups: + typescript-client-deps: + patterns: ["*"] + typescript-client-security: + applies-to: security-updates + patterns: ["*"] + + # Maintain dependencies for multiplayer + - package-ecosystem: "npm" + directory: "/multiplayer" + schedule: + interval: "weekly" + groups: + multiplayer-deps: + patterns: ["*"] + multiplayer-security: + applies-to: security-updates + patterns: ["*"] diff --git a/.github/workflows/ai-agent-tests.yml b/.github/workflows/ai-agent-tests.yml index cd07602109..8a3e9e6dbc 100644 --- a/.github/workflows/ai-agent-tests.yml +++ b/.github/workflows/ai-agent-tests.yml @@ -63,11 +63,11 @@ jobs: with: bun-version: 1.4.0 - - uses: actions/setup-node@v4 + - uses: actions/setup-node@v7 with: node-version: "20" - - uses: actions/setup-python@v5 + - uses: actions/setup-python@v7 with: python-version: "3.11" diff --git a/.github/workflows/ai-evals-test.yml b/.github/workflows/ai-evals-test.yml index 87eefab0f3..573bc77df9 100644 --- a/.github/workflows/ai-evals-test.yml +++ b/.github/workflows/ai-evals-test.yml @@ -77,7 +77,7 @@ jobs: with: bun-version: 1.4.0 - - uses: actions/setup-node@v4 + - uses: actions/setup-node@v7 with: # Node 24 ships npm 11, which frontend/package-lock.json is authored # with; npm 10 rejects it ("Missing: picomatch@4.0.5 from lock file"). diff --git a/.github/workflows/backend-test-windows.yml b/.github/workflows/backend-test-windows.yml index f9153c0ce0..54a62ec495 100644 --- a/.github/workflows/backend-test-windows.yml +++ b/.github/workflows/backend-test-windows.yml @@ -67,7 +67,7 @@ jobs: # tests what production runs, instead of floating on the latest v2.x. deno-version: 2.2.1 - - uses: actions/setup-go@v2 + - uses: actions/setup-go@v7 with: go-version: 1.21.5 @@ -75,11 +75,11 @@ jobs: with: bun-version: 1.4.0 - - uses: actions/setup-node@v4 + - uses: actions/setup-node@v7 with: node-version: "20" - - uses: astral-sh/setup-uv@v6.2.1 + - uses: astral-sh/setup-uv@v7 with: version: "0.11.24" diff --git a/.github/workflows/backend-test.yml b/.github/workflows/backend-test.yml index 9f65c2ae47..fea84e7781 100644 --- a/.github/workflows/backend-test.yml +++ b/.github/workflows/backend-test.yml @@ -53,16 +53,16 @@ jobs: # Pin to the Deno version shipped in the runtime image (Dockerfile) so CI # tests what production runs, instead of floating on the latest v2.x. deno-version: 2.2.1 - - uses: actions/setup-go@v2 + - uses: actions/setup-go@v7 with: go-version: 1.21.5 - uses: oven-sh/setup-bun@v2 with: bun-version: 1.4.0 - - uses: actions/setup-node@v4 + - uses: actions/setup-node@v7 with: node-version: "24" - - uses: astral-sh/setup-uv@v6.2.1 + - uses: astral-sh/setup-uv@v7 with: version: "0.11.24" - uses: shivammathur/setup-php@v2 diff --git a/.github/workflows/check-docs-links.yml b/.github/workflows/check-docs-links.yml index ec17c06771..e32e976f0c 100644 --- a/.github/workflows/check-docs-links.yml +++ b/.github/workflows/check-docs-links.yml @@ -16,7 +16,7 @@ jobs: sparse-checkout: | frontend/src .github/scripts - - uses: actions/setup-node@v4 + - uses: actions/setup-node@v7 with: node-version: "22.x" - name: Verify docs links are not 404 diff --git a/.github/workflows/check-system-prompts.yml b/.github/workflows/check-system-prompts.yml index 3e684c82f9..370a177716 100644 --- a/.github/workflows/check-system-prompts.yml +++ b/.github/workflows/check-system-prompts.yml @@ -28,7 +28,7 @@ jobs: steps: - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 + - uses: actions/setup-python@v7 with: python-version: "3.12" diff --git a/.github/workflows/cli-tests.yml b/.github/workflows/cli-tests.yml index e26231eda4..19afa777ae 100644 --- a/.github/workflows/cli-tests.yml +++ b/.github/workflows/cli-tests.yml @@ -43,7 +43,7 @@ jobs: uses: actions/checkout@v4 - name: Setup Node.js - uses: actions/setup-node@v4 + uses: actions/setup-node@v7 with: node-version: "20" @@ -89,7 +89,7 @@ jobs: cache-workspaces: backend - name: Setup Node.js - uses: actions/setup-node@v4 + uses: actions/setup-node@v7 with: node-version: "20" @@ -143,7 +143,7 @@ jobs: cache-workspaces: backend - name: Setup Node.js - uses: actions/setup-node@v4 + uses: actions/setup-node@v7 with: node-version: "20" diff --git a/.github/workflows/codex-pr-review.yml b/.github/workflows/codex-pr-review.yml index 7a3882df7c..354e8a60ca 100644 --- a/.github/workflows/codex-pr-review.yml +++ b/.github/workflows/codex-pr-review.yml @@ -213,7 +213,7 @@ jobs: - name: Set up Node.js if: steps.codex_config.outputs.enabled == 'true' && steps.pr.outputs.skip != 'true' - uses: actions/setup-node@v4 + uses: actions/setup-node@v7 with: node-version: 22 diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml index a8b20623bc..b468387057 100644 --- a/.github/workflows/docker-image.yml +++ b/.github/workflows/docker-image.yml @@ -40,6 +40,11 @@ jobs: runs-on: ubicloud if: (github.event_name != 'workflow_dispatch') || (github.event.inputs && !github.event.inputs.ee) + # Jobs that build from, extract or retag this image must reference it by + # this digest: concurrent main-branch runs push :dev too, so a tag run + # reading :dev can ship a main build under the release version. + outputs: + digest: ${{ steps.docker_build.outputs.digest }} steps: - uses: actions/checkout@v4 with: @@ -112,6 +117,8 @@ jobs: build_ee: runs-on: ubicloud if: (github.event_name != 'workflow_dispatch') || github.event.inputs.ee + outputs: + digest: ${{ steps.docker_build.outputs.digest }} steps: - uses: actions/checkout@v4 with: @@ -192,31 +199,31 @@ jobs: - uses: actions/checkout@v4 - run: | # pulling docker image with desired arch so that actions-docker-extract doesn't do it - docker pull --platform "linux/$ARCH" ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ env.DEV_SHA }} - docker pull --platform "linux/$ARCH" ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:${{ env.DEV_SHA }} + docker pull --platform "linux/$ARCH" ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}@${{ needs.build.outputs.digest }} + docker pull --platform "linux/$ARCH" ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee@${{ needs.build_ee.outputs.digest }} - run: | # Checks the image is in docker prior to running actions-docker-extract. It fails if not # Also useful to visually check that the arch is the right opencontainers - docker image inspect ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ env.DEV_SHA }} - docker image inspect ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:${{ env.DEV_SHA }} + docker image inspect ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}@${{ needs.build.outputs.digest }} + docker image inspect ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee@${{ needs.build_ee.outputs.digest }} - uses: shrink/actions-docker-extract@v3 id: extract with: - image: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ env.DEV_SHA }} + image: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}@${{ needs.build.outputs.digest }} path: "/usr/src/app/windmill" - uses: shrink/actions-docker-extract@v3 id: extract-duckdb-ffi-internal with: - image: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ env.DEV_SHA }} + image: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}@${{ needs.build.outputs.digest }} path: "/usr/src/app/libwindmill_duckdb_ffi_internal.so" - uses: shrink/actions-docker-extract@v3 id: extract-ee with: - image: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:${{ env.DEV_SHA }} + image: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee@${{ needs.build_ee.outputs.digest }} path: "/usr/src/app/windmill" - name: Rename binary with corresponding architecture @@ -373,8 +380,8 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} - name: Tag main and latest run: | - docker buildx imagetools create ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ env.DEV_SHA }} --tag ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest - docker buildx imagetools create ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ env.DEV_SHA }} --tag ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:main + docker buildx imagetools create ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}@${{ needs.build.outputs.digest }} --tag ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest + docker buildx imagetools create ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}@${{ needs.build.outputs.digest }} --tag ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:main - uses: sigstore/cosign-installer@v4.1.2 if: startsWith(github.ref, 'refs/tags/v') @@ -409,8 +416,8 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} - name: Tag main and latest for ee run: | - docker buildx imagetools create ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:${{ env.DEV_SHA }} --tag ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:latest - docker buildx imagetools create ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:${{ env.DEV_SHA }} --tag ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:main + docker buildx imagetools create ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee@${{ needs.build_ee.outputs.digest }} --tag ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:latest + docker buildx imagetools create ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee@${{ needs.build_ee.outputs.digest }} --tag ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:main - uses: sigstore/cosign-installer@v4.1.2 if: startsWith(github.ref, 'refs/tags/v') @@ -549,6 +556,8 @@ jobs: push: true sbom: ${{ startsWith(github.ref, 'refs/tags/v') && github.event_name == 'push' }} file: "./docker/DockerfileCuda" + build-args: | + BASE_IMAGE=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee@${{ needs.build_ee.outputs.digest }} tags: | ${{ steps.meta-ee-public.outputs.tags }} labels: | @@ -602,6 +611,8 @@ jobs: push: true sbom: ${{ startsWith(github.ref, 'refs/tags/v') && github.event_name == 'push' }} file: "./docker/DockerfileSlim" + build-args: | + BASE_IMAGE=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}@${{ needs.build.outputs.digest }} tags: | ${{ steps.meta-ee-public.outputs.tags }} labels: | @@ -656,6 +667,8 @@ jobs: push: true sbom: ${{ startsWith(github.ref, 'refs/tags/v') && github.event_name == 'push' }} file: "./docker/DockerfileSlimEe" + build-args: | + BASE_IMAGE=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee@${{ needs.build_ee.outputs.digest }} tags: | ${{ steps.meta-ee-public.outputs.tags }} labels: | @@ -709,6 +722,8 @@ jobs: push: true sbom: ${{ startsWith(github.ref, 'refs/tags/v') && github.event_name == 'push' }} file: "./docker/DockerfileFull" + build-args: | + BASE_IMAGE=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}@${{ needs.build.outputs.digest }} tags: | ${{ steps.meta-public.outputs.tags }} labels: | @@ -761,6 +776,8 @@ jobs: push: true sbom: ${{ startsWith(github.ref, 'refs/tags/v') && github.event_name == 'push' }} file: "./docker/DockerfileFullEe" + build-args: | + BASE_IMAGE=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee@${{ needs.build_ee.outputs.digest }} tags: | ${{ steps.meta-ee-public.outputs.tags }} labels: | diff --git a/.github/workflows/frontend-check.yml b/.github/workflows/frontend-check.yml index 178ed53dc5..6347614e16 100644 --- a/.github/workflows/frontend-check.yml +++ b/.github/workflows/frontend-check.yml @@ -18,7 +18,7 @@ jobs: runs-on: ubicloud-standard-8 steps: - uses: actions/checkout@v4 - - uses: actions/setup-node@v5 + - uses: actions/setup-node@v7 with: node-version: 24 cache: "npm" diff --git a/.github/workflows/git-sync-test.yml b/.github/workflows/git-sync-test.yml index bd15ec8876..b19f899a20 100644 --- a/.github/workflows/git-sync-test.yml +++ b/.github/workflows/git-sync-test.yml @@ -141,7 +141,7 @@ jobs: with: deno-version: v2.x - - uses: actions/setup-node@v4 + - uses: actions/setup-node@v7 with: node-version: "20" diff --git a/.github/workflows/go_on_release.yml b/.github/workflows/go_on_release.yml index 78ce12c6c3..6e53db09e7 100644 --- a/.github/workflows/go_on_release.yml +++ b/.github/workflows/go_on_release.yml @@ -13,7 +13,7 @@ jobs: runs-on: ubicloud-standard-8 steps: - uses: actions/checkout@v4 - - uses: actions/setup-go@v4 + - uses: actions/setup-go@v7 - name: install_jq run: | sudo apt-get update diff --git a/.github/workflows/npm_on_release.yml b/.github/workflows/npm_on_release.yml index 7d30265c74..631ab53153 100644 --- a/.github/workflows/npm_on_release.yml +++ b/.github/workflows/npm_on_release.yml @@ -10,7 +10,7 @@ jobs: runs-on: ubicloud-standard-8 steps: - uses: actions/checkout@v4 - - uses: actions/setup-node@v3 + - uses: actions/setup-node@v7 with: node-version: "20.x" registry-url: "https://registry.npmjs.org" @@ -21,7 +21,7 @@ jobs: runs-on: ubicloud-standard-8 steps: - uses: actions/checkout@v4 - - uses: actions/setup-node@v3 + - uses: actions/setup-node@v7 with: node-version: "20.x" registry-url: "https://registry.npmjs.org" @@ -32,7 +32,7 @@ jobs: runs-on: ubicloud-standard-8 steps: - uses: actions/checkout@v4 - - uses: actions/setup-node@v3 + - uses: actions/setup-node@v7 with: node-version: "20.x" registry-url: "https://registry.npmjs.org" diff --git a/.github/workflows/pi-pr-review.yml b/.github/workflows/pi-pr-review.yml index 9b536b6ed5..a89275cfb5 100644 --- a/.github/workflows/pi-pr-review.yml +++ b/.github/workflows/pi-pr-review.yml @@ -206,7 +206,7 @@ jobs: - name: Set up Node.js if: steps.pi_config.outputs.enabled == 'true' && steps.pr.outputs.skip != 'true' - uses: actions/setup-node@v4 + uses: actions/setup-node@v7 with: node-version: 22 diff --git a/.github/workflows/publish-cli-docs.yml b/.github/workflows/publish-cli-docs.yml index 9e76117eb5..d6a2bdb332 100644 --- a/.github/workflows/publish-cli-docs.yml +++ b/.github/workflows/publish-cli-docs.yml @@ -38,7 +38,7 @@ jobs: ssh-key: ${{ secrets.CLI_DOCS_DEPLOY_KEY }} fetch-depth: 0 - - uses: actions/setup-python@v5 + - uses: actions/setup-python@v7 with: python-version: "3.12" diff --git a/.github/workflows/sdk-tests.yml b/.github/workflows/sdk-tests.yml index 6f7d79c6af..260fbb3a02 100644 --- a/.github/workflows/sdk-tests.yml +++ b/.github/workflows/sdk-tests.yml @@ -43,7 +43,7 @@ jobs: with: bun-version: latest - - uses: actions/setup-node@v4 + - uses: actions/setup-node@v7 with: node-version: "20.x" @@ -58,7 +58,7 @@ jobs: uses: actions/checkout@v4 - name: Setup uv - uses: astral-sh/setup-uv@v5 + uses: astral-sh/setup-uv@v7 # The interpreter is named explicitly: on a clean checkout uv picks the # runner's system python and stops with "not compatible with the locked diff --git a/.github/workflows/validate-openapi.yml b/.github/workflows/validate-openapi.yml index 5bc59ebc4c..846621b503 100644 --- a/.github/workflows/validate-openapi.yml +++ b/.github/workflows/validate-openapi.yml @@ -15,7 +15,7 @@ jobs: uses: actions/checkout@v4 - name: Set up Node.js - uses: actions/setup-node@v4 + uses: actions/setup-node@v7 with: node-version: '20' diff --git a/.github/workflows/yaml-validator-tests.yml b/.github/workflows/yaml-validator-tests.yml index 04d31451d4..c5524312cb 100644 --- a/.github/workflows/yaml-validator-tests.yml +++ b/.github/workflows/yaml-validator-tests.yml @@ -24,7 +24,7 @@ jobs: steps: - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 + - uses: actions/setup-node@v7 with: node-version: "20" diff --git a/.release-please-manifest.json b/.release-please-manifest.json index 23ae9b3ffa..6dc5574962 100644 --- a/.release-please-manifest.json +++ b/.release-please-manifest.json @@ -1,3 +1,3 @@ { - ".": "1.815.0" + ".": "1.817.0" } diff --git a/AGENTS.md b/AGENTS.md index a315f1a67c..77b38c3267 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -55,12 +55,23 @@ Open-source platform for internal tools, workflows, API integrations, background > defaults in this section apply only to a plain single checkout. **Discover the real > values before running anything** — see "Per-worktree ports and database" below. -**Check whether they are already running before starting anything.** In a webmux worktree -(`$WEBMUX_WORKTREE_PATH` is set) the backend and frontend are already up in sibling tmux panes — -use those, don't spawn your own. `tmux list-panes -t "$(tmux display-message -p -t "$TMUX_PANE" -'#{window_id}')" -F '#{pane_index} #{pane_current_command}'` shows what is running; read its log -with `tmux capture-pane`, and see `backend/AGENTS.md` to restart it with different cargo features. -A second server started in your own shell fights the first one for the port. The commands below +**Check whether they are already running before starting anything.** In a managed worktree the +backend and frontend are already up in sibling panes — use those, don't spawn your own. A second +server started in your own shell fights the first one for the port. + +**herdr** is the worktree manager; `$HERDR_ENV=1` marks one of its panes. `herdr pane list +--workspace "$HERDR_WORKSPACE_ID"` lists this worktree's panes — match on `cwd`, since the servers +run from `backend/` and `frontend/` — and `herdr pane read --source recent-unwrapped +--lines 50` reads one's log; `herdr --skill` prints the full reference for +inspecting and driving panes and agents. The plugins that provision these worktrees live in the +private `windmill-labs/windmill-herdr` — clone it and run `./setup.sh` to install them and the +keybindings they need. + +A worktree from the older **webmux** setup sets `$WEBMUX_WORKTREE_PATH` instead and is driven +through tmux: `tmux list-panes -t "$(tmux display-message -p -t "$TMUX_PANE" '#{window_id}')" -F +'#{pane_index} #{pane_current_command}'` for what is running, `tmux capture-pane` for a pane's log. + +See `backend/AGENTS.md` to restart the backend with different cargo features. The commands below are for a plain checkout with nothing running. - **Backend**: `cargo run` from `backend/` (API at http://localhost:8000) @@ -72,10 +83,16 @@ are for a plain checkout with nothing running. ### Per-worktree ports and database -In a webmux worktree the authoritative values live in -`$(git rev-parse --git-dir)/webmux/runtime.env` — `BACKEND_PORT`, `FRONTEND_PORT`, -`DATABASE_URL`, `CARGO_FEATURES`, `WM_DB_NAME`. Every pane sources it at startup. Read that -first: it is not a `.env*` file, so the repo's secret-file read rules don't stand in the way. +**`.env.local` in the worktree root holds the real values** — `BACKEND_PORT`, `FRONTEND_PORT`, +`REMOTE`, `DATABASE_URL`, `CARGO_FEATURES`, `WM_DB_NAME`. Every manager writes those fields, so it +is correct whichever one you are in. Read it with `cat .env.local` from a shell: the file-read +tool denies every `.env.*` path, and `DATABASE_URL` is written with an `export` prefix that a +`^DATABASE_URL=` grep misses. + +herdr keeps nothing besides that file; its hooks write it and read it back. A webmux worktree +additionally has `$(git rev-parse --git-dir)/webmux/runtime.env`, which every pane sources at +startup and which carries the extras `.env.local` lacks: `WEBMUX_*`, `WM_CLONE_DB`, +`USE_RUST_PLUGIN`. In a plain checkout, fall back to `.env` / `.env.local` (repo root) and `backend/.env`. @@ -84,19 +101,25 @@ hook. It is not a copy of the main dev instance: you get the `admins` workspace, `admin@windmill.dev` superadmin, the license key copied from the base database, and whatever the migrations seed — and none of your own workspaces, scripts, flows or apps. Create whatever a test needs. Cloning the base `windmill` database instead is -opt-in per project via `WM_CLONE_DB` in `.webmux.yaml`; read the note there before turning it on. +opt-in via `WM_CLONE_DB` (the `windmill.worktree` plugin's `config.env` under herdr, or +`.webmux.yaml` under webmux); read the note in `.webmux.yaml` before turning it on. The database is named after the **worktree directory, not the branch** (`scripts/worktree-common.sh`): `windmill_` + the directory basename with `-` → `_`, which Postgres then truncates at 63 -characters. Branch `hugo/win-2340-ai-agent-evals-standalone-agent-runs-and-eval-datasets` sits in -a worktree directory named `win-2340-…`, so its database is -`windmill_win_2340_ai_agent_evals_standalone_agent_runs_and_eval` — no `hugo_`, and the tail -chopped. Take `WM_DB_NAME` from `runtime.env` instead of reconstructing the name. Read those, or -discover from what is already running: +characters. herdr creates the directory under `~/.herdr/worktrees//` from the branch name +with each `/` turned into `-`, so branch +`hugo/win-2544-add-options-field-to-the-postgresql-resource-type` gets the database +`windmill_hugo_win_2544_add_options_field_to_the_postgresql_reso` — the whole branch name, cut +mid-word at the limit. The two drift apart as soon as the branch is renamed, and webmux named its +directories after the ticket alone, so reconstructing the name from the branch you are on is +wrong in both. Take `WM_DB_NAME` from `.env.local` instead. Read that, or discover from what is +already running: ```bash +# match on the worktree directory, truncated the way Postgres truncates it (63 - len('windmill_')): psql postgres://postgres:changeme@localhost:5432/postgres -tAc \ - "select datname from pg_database where datname like 'windmill%'" | grep "$(git branch --show-current | tr - _)" + "select datname from pg_database where datname like 'windmill%'" \ + | grep -F "$(basename "$(git rev-parse --show-toplevel)" | tr '/-' '_' | cut -c1-54)" # the port the frontend actually proxies to (REMOTE of this worktree's vite): for p in $(pgrep -f vite); do case "$(readlink /proc/$p/cwd)" in *"$(basename "$(git rev-parse --show-toplevel)")"*) tr '\0' '\n' < /proc/$p/environ | grep -E '^REMOTE=|^PORT=';; esac; done diff --git a/CHANGELOG.md b/CHANGELOG.md index 820f2c5d6c..595426d619 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,59 @@ # Changelog +## [1.817.0](https://github.com/windmill-labs/windmill/compare/v1.816.0...v1.817.0) (2026-09-22) + + +### Features + +* add workspace mentions to AI composer ([#11264](https://github.com/windmill-labs/windmill/issues/11264)) ([e7ba595](https://github.com/windmill-labs/windmill/commit/e7ba5954b69514d72c140f9b1e697fb2236df23e)) +* allow workspace-scoped dynamic custom tags ([#11283](https://github.com/windmill-labs/windmill/issues/11283)) ([efa7a0a](https://github.com/windmill-labs/windmill/commit/efa7a0a70af43f8827542059dd450326a9b88d62)) +* check dynamic worker tags on the tag they resolve to ([#11271](https://github.com/windmill-labs/windmill/issues/11271)) ([69fafb9](https://github.com/windmill-labs/windmill/commit/69fafb9262f90184e6c2dc9a84f1c3feeb9d9638)) +* deploy from the UI to a workspace on another instance ([#11245](https://github.com/windmill-labs/windmill/issues/11245)) ([4d12ea4](https://github.com/windmill-labs/windmill/commit/4d12ea461454f89367bd91daa7e8c39a89035507)) + + +### Bug Fixes + +* add missing credential fields to settings redaction lists ([#11272](https://github.com/windmill-labs/windmill/issues/11272)) ([2a40f25](https://github.com/windmill-labs/windmill/commit/2a40f25bda81d41b9f578338c85656744f59ebfc)) +* apply SSRF validation to workspace webhook URLs ([#11285](https://github.com/windmill-labs/windmill/issues/11285)) ([586699c](https://github.com/windmill-labs/windmill/commit/586699c4938b5203920a43e69a706310078b93f5)) +* apply token path scopes to the native trigger list ([#11281](https://github.com/windmill-labs/windmill/issues/11281)) ([faf7b22](https://github.com/windmill-labs/windmill/commit/faf7b22be0827738c90c1f3ed6cd3e214d6c63d3)) +* check workflow-as-code task tags against CUSTOM_TAGS ([#11273](https://github.com/windmill-labs/windmill/issues/11273)) ([2699d50](https://github.com/windmill-labs/windmill/commit/2699d5065ea3dfe985636733fb6022ca23187291)) +* clear the impersonation token from session storage on logout ([#11274](https://github.com/windmill-labs/windmill/issues/11274)) ([9936673](https://github.com/windmill-labs/windmill/commit/9936673ee62fb6dd929a654a6138aeca687c0fb7)) +* enforce token path scopes on GET /raw_apps/list ([#11284](https://github.com/windmill-labs/windmill/issues/11284)) ([9ad2c91](https://github.com/windmill-labs/windmill/commit/9ad2c91ddb661ab60ada9ec4039ef8d025602969)) +* keep a cancel that lands after a worker last read its queue row ([#11246](https://github.com/windmill-labs/windmill/issues/11246)) ([02006f3](https://github.com/windmill-labs/windmill/commit/02006f364a4217e967435d9e71d8617c03b65413)) +* size the AI chat output budget by model, not provider ([#11286](https://github.com/windmill-labs/windmill/issues/11286)) ([7d0f2ce](https://github.com/windmill-labs/windmill/commit/7d0f2ce3bf920a68f1ea22d59481d239947f1b51)) +* stop cutting DeepSeek chat turns off mid-thought ([#11282](https://github.com/windmill-labs/windmill/issues/11282)) ([5821700](https://github.com/windmill-labs/windmill/commit/58217006ece56bbaf0447af67cac487e72b928c7)) +* stop the raw app editor stealing focus in ai sessions ([#11287](https://github.com/windmill-labs/windmill/issues/11287)) ([7322143](https://github.com/windmill-labs/windmill/commit/732214361f36eb07d65854118bd053c96ce681fb)) + +## [1.816.0](https://github.com/windmill-labs/windmill/compare/v1.815.0...v1.816.0) (2026-09-21) + + +### Features + +* add an ACL editor for data table roles ([#11063](https://github.com/windmill-labs/windmill/issues/11063)) ([9994e03](https://github.com/windmill-labs/windmill/commit/9994e03bc75404503001a5db37e175b66e7371f2)) +* add per-model context window override in workspace AI settings ([#11232](https://github.com/windmill-labs/windmill/issues/11232)) ([c637b06](https://github.com/windmill-labs/windmill/commit/c637b06a548a5725aceb7504bcb62383d35614ac)) +* **ai-chat:** disclose an app's exposure on deploy, refuse drag-and-drop apps ([#11137](https://github.com/windmill-labs/windmill/issues/11137)) ([85b36a2](https://github.com/windmill-labs/windmill/commit/85b36a2de5f3fa08750b6d2ab8203f4b745cfdd2)) +* data table roles in the DB manager and raw apps ([#11139](https://github.com/windmill-labs/windmill/issues/11139)) ([e1e3692](https://github.com/windmill-labs/windmill/commit/e1e3692fbc82d50c021ef8bf8ca7019d760705f0)) +* drafts follow their item through a move; behind means base ≠ head ([#10577](https://github.com/windmill-labs/windmill/issues/10577)) ([f00b2fc](https://github.com/windmill-labs/windmill/commit/f00b2fcb1e65cb8f8e95d55f7cc58b816ac7d89d)) +* edit variables, resources and triggers in their own session tab ([#11206](https://github.com/windmill-labs/windmill/issues/11206)) ([6186a06](https://github.com/windmill-labs/windmill/commit/6186a0645d3f3e92e06b666a6b5774b8af2b488f)) +* let every EE instance set the announcement banner ([#11238](https://github.com/windmill-labs/windmill/issues/11238)) ([87761d1](https://github.com/windmill-labs/windmill/commit/87761d179799026ebaf9ed91bf589852b5453ee3)) +* snap-collapse the sidebar by dragging it to the left edge ([#11225](https://github.com/windmill-labs/windmill/issues/11225)) ([56a6ad3](https://github.com/windmill-labs/windmill/commit/56a6ad322e8354695a0a96eef9e83ffa135b3ad4)) + + +### Bug Fixes + +* bump git sync hub scripts to cli 1.815.1 ([#11239](https://github.com/windmill-labs/windmill/issues/11239)) ([9e0e835](https://github.com/windmill-labs/windmill/commit/9e0e835c6876305d3a44902fb1432fd9c2348a7c)) +* check custom_instance_user can connect during instance database setup ([#11244](https://github.com/windmill-labs/windmill/issues/11244)) ([1b0f119](https://github.com/windmill-labs/windmill/commit/1b0f119c7b8ac2aacf2e2b4428c57f661df87ae5)) +* **cli:** read wmill.yaml from the branch a git-sync deploy writes to ([#11236](https://github.com/windmill-labs/windmill/issues/11236)) ([974b026](https://github.com/windmill-labs/windmill/commit/974b0265f85f9cfac65bbd2f6ebbcb0e966e3301)) +* guard data table migration routes against operators and unauthorized authors ([#11243](https://github.com/windmill-labs/windmill/issues/11243)) ([b742544](https://github.com/windmill-labs/windmill/commit/b7425443e1e08be029fc676047f727f84b646323)) +* tell the user when a draft conflict has stopped their edits saving ([#11241](https://github.com/windmill-labs/windmill/issues/11241)) ([d8ebbc0](https://github.com/windmill-labs/windmill/commit/d8ebbc06f5c56dad2570115700d640105d856c57)) + + +### Performance Improvements + +* index audit logs by workspace and operation ([#11235](https://github.com/windmill-labs/windmill/issues/11235)) ([3b8c326](https://github.com/windmill-labs/windmill/commit/3b8c3264ecb33884420b1bdac9cae9c841e721a0)) +* keep the sessions page fast with many ai sessions ([#11224](https://github.com/windmill-labs/windmill/issues/11224)) ([4cbc944](https://github.com/windmill-labs/windmill/commit/4cbc944c0bcb938300c964437d745d15a3ea608f)) +* retire the legacy pre-partitioning audit table ([#11240](https://github.com/windmill-labs/windmill/issues/11240)) ([9f2685d](https://github.com/windmill-labs/windmill/commit/9f2685d788e624c9ed9db5cbaee456ea21af2b29)) + ## [1.815.0](https://github.com/windmill-labs/windmill/compare/v1.814.0...v1.815.0) (2026-09-18) diff --git a/ai_evals/adapters/frontend/mockBackend.ts b/ai_evals/adapters/frontend/mockBackend.ts index 950cc2e1b5..90df0cf870 100644 --- a/ai_evals/adapters/frontend/mockBackend.ts +++ b/ai_evals/adapters/frontend/mockBackend.ts @@ -14,7 +14,6 @@ import type { DataMetric, DataTableTables, DataTableTableSchema, - EndpointTool, GetDraftForUserResponse, GetOwnDraftResponse, ListDraftsResponse, @@ -1020,119 +1019,6 @@ function buildBenchmarkApp(app: BenchmarkWorkspaceApp): AppWithLastVersion { } } -// ============= API endpoint catalog (McpService.listMcpTools + raw fetch) ============= -// The global chat's API catalog tools list endpoints via McpService and execute -// them with a plain relative fetch('/api/...'), which has no meaning in the -// vitest environment. A representative slice of the real catalog is served here, -// and `handleBenchmarkApiFetch` answers the executed calls. - -const BENCHMARK_MCP_TOOLS: EndpointTool[] = [ - { - name: 'listWorkers', - description: 'List workers', - instructions: 'List all workers with their last ping and job counts.', - path: '/workers/list', - method: 'GET', - query_params_schema: { - type: 'object', - properties: { page: { type: 'integer' }, per_page: { type: 'integer' } } - } - }, - { - name: 'listQueue', - description: 'List queued jobs', - instructions: '', - path: '/w/{workspace}/jobs/queue/list', - method: 'GET', - path_params_schema: { - type: 'object', - properties: { workspace: { type: 'string' } }, - required: ['workspace'] - } - }, - { - name: 'getJob', - description: 'get job', - instructions: '', - path: '/w/{workspace}/jobs_u/get/{id}', - method: 'GET', - path_params_schema: { - type: 'object', - properties: { workspace: { type: 'string' }, id: { type: 'string', format: 'uuid' } }, - required: ['workspace', 'id'] - }, - query_params_schema: { - type: 'object', - properties: { - no_logs: { type: 'boolean' }, - no_code: { type: 'boolean' }, - approval_token: { type: 'string' } - }, - required: [] - } - }, - { - name: 'runScriptByPath', - description: 'Run the deployed version of a script by path', - instructions: '', - path: '/w/{workspace}/jobs/run/p/{path}', - method: 'POST', - path_params_schema: { - type: 'object', - properties: { workspace: { type: 'string' }, path: { type: 'string' } }, - required: ['workspace', 'path'] - }, - body_schema: { type: 'object', properties: {} } - }, - { - name: 'runFlowByPath', - description: 'Run the deployed version of a flow by path', - instructions: '', - path: '/w/{workspace}/jobs/run/f/{path}', - method: 'POST', - path_params_schema: { - type: 'object', - properties: { workspace: { type: 'string' }, path: { type: 'string' } }, - required: ['workspace', 'path'] - }, - body_schema: { type: 'object', properties: {} } - }, - // Draft-covered endpoints, present so steering cases exercise the guard the - // way production does (hidden from search, refused at call time). - { - name: 'getScriptByPath', - description: 'Get a script by path', - instructions: '', - path: '/w/{workspace}/scripts/get/p/{path}', - method: 'GET' - }, - { - name: 'createFlow', - description: 'Create a flow', - instructions: '', - path: '/w/{workspace}/flows/create', - method: 'POST' - }, - { - name: 'deleteSchedule', - description: 'Delete a schedule', - instructions: '', - path: '/w/{workspace}/schedules/delete/{path}', - method: 'DELETE' - }, - { - name: 'getVariable', - description: 'Get a variable', - instructions: '', - path: '/w/{workspace}/variables/get/{path}', - method: 'GET' - } -] - -export function listBenchmarkMcpTools(): EndpointTool[] { - return BENCHMARK_MCP_TOOLS -} - /** A stand-in Windmill hub. `search_hub_scripts` and a `hub/` read go out over * relative `/api/...` fetches, which have no origin here, so without these the * hub tools throw and no case can exercise hub reuse. Serving fixtures rather @@ -1292,26 +1178,6 @@ const BENCHMARK_WORKERS = [ } ] -const BENCHMARK_JOB_GET_PATH = /^\/api\/w\/([^/]+)\/jobs_u\/get\/([^/]+)$/ -const BENCHMARK_RUN_BY_PATH = /^\/api\/w\/([^/]+)\/jobs\/run\/(p|f)\/([^/]+)$/ - -/** `executeEndpoint` sends a JSON string; anything else means no args were supplied. */ -function parseBenchmarkRequestBody( - body: BodyInit | null | undefined -): Record | undefined { - if (typeof body !== 'string') { - return undefined - } - try { - const parsed = JSON.parse(body) - return typeof parsed === 'object' && parsed !== null - ? (parsed as Record) - : undefined - } catch { - return undefined - } -} - /** True when `handleBenchmarkApiFetch` has an answer for this `/api/...` url. * Any other relative fetch must keep its normal (non-benchmark) behavior — * intercepting it with a synthetic 404 sends the model into retry loops. */ @@ -1323,17 +1189,13 @@ export function hasBenchmarkApiHandler(url: string): boolean { const path = url.split('?')[0] return ( path === '/api/workers/list' || - BENCHMARK_JOB_GET_PATH.test(path) || - BENCHMARK_RUN_BY_PATH.test(path) || - /^\/api\/w\/[^/]+\/jobs\/queue\/list$/.test(path) || path === '/api/embeddings/query_hub_scripts' || path.startsWith('/api/scripts/hub/get_full/') || BENCHMARK_AI_MODELS_PATH.test(path) ) } -/** Answer a relative `/api/...` fetch — from the API catalog executor, or from the - * chat's hub tools. */ +/** Answer the relative `/api/...` fetches no mocked service covers. */ export function handleBenchmarkApiFetch(url: string, init?: RequestInit): Response { const path = url.split('?')[0] if (path === '/api/workers/list') { @@ -1350,39 +1212,6 @@ export function handleBenchmarkApiFetch(url: string, init?: RequestInit): Respon ?.aiProviders?.find((entry) => entry.path === resourcePath) return Response.json({ data: (seed?.models ?? []).map((id) => ({ id })) }) } - if (/^\/api\/w\/[^/]+\/jobs\/queue\/list$/.test(path)) { - return Response.json([]) - } - const jobGet = BENCHMARK_JOB_GET_PATH.exec(path) - if (jobGet) { - const id = decodeURIComponent(jobGet[2]) - const job = getBenchmarkCompletedJob(decodeURIComponent(jobGet[1]), id) - if (!job) { - return Response.json({ error: `Job not found for "${id}"` }, { status: 404 }) - } - // The real endpoint lets a caller drop the bulky fields. Ignoring that here would - // size the model's context off a payload it explicitly asked to shrink. - const query = new URLSearchParams(url.split('?')[1] ?? '') - if (query.get('no_logs') === 'true') { - delete job.logs - } - if (query.get('no_code') === 'true') { - delete job.raw_code - } - return Response.json(job) - } - const runByPath = BENCHMARK_RUN_BY_PATH.exec(path) - if (runByPath) { - const workspace = decodeURIComponent(runByPath[1]) - const runnablePath = decodeURIComponent(runByPath[3]) - const args = parseBenchmarkRequestBody(init?.body) - // The real endpoint answers with the bare job id as text, not JSON. - return new Response( - runByPath[2] === 'f' - ? runBenchmarkFlowByPath({ workspace, path: runnablePath, args }) - : runBenchmarkScriptByPath({ workspace, path: runnablePath, args }) - ) - } if (path === '/api/embeddings/query_hub_scripts') { const text = new URLSearchParams(url.split('?')[1] ?? '').get('text') ?? '' return Response.json(searchBenchmarkHubScripts(text)) diff --git a/ai_evals/adapters/frontend/mockBackendApi.test.ts b/ai_evals/adapters/frontend/mockBackendApi.test.ts index d9cfa08bf4..85bd1f3320 100644 --- a/ai_evals/adapters/frontend/mockBackendApi.test.ts +++ b/ai_evals/adapters/frontend/mockBackendApi.test.ts @@ -1,83 +1,18 @@ -import { afterEach, beforeEach, describe, expect, it } from 'bun:test' -import { - createBenchmarkCompletedJob, - getBenchmarkCompletedJob, - handleBenchmarkApiFetch, - hasBenchmarkApiHandler, - listBenchmarkMcpTools, - resetBenchmarkMockBackend, - registerBenchmarkWorkspaceRunnables -} from './mockBackend' +import { describe, expect, it } from 'bun:test' +import { handleBenchmarkApiFetch, hasBenchmarkApiHandler } from './mockBackend' -const WORKSPACE = 'benchmark-api-ws' +// `list_workers`'s client call is not faked, so its request really does leave as a relative +// `/api/...` url that node's fetch cannot resolve. Without this route the tool throws instead +// of answering. +describe('benchmark API fetch handlers', () => { + it('lists workers, the way list_workers reaches them', async () => { + const url = '/api/workers/list?per_page=100' -// A catalog entry with no fetch handler is a dead end: the catalog executor builds a -// relative `/api/...` url, the stub declines it, and node's fetch throws on the relative -// url instead of returning a result the model can act on. Mutating entries are reachable -// too — the eval runners define no `requestConfirmation`, so `call_api_endpoint` executes -// unconfirmed. -describe('benchmark API catalog', () => { - beforeEach(() => resetBenchmarkMockBackend()) - afterEach(() => resetBenchmarkMockBackend()) + expect(hasBenchmarkApiHandler(url)).toBe(true) + const body = await handleBenchmarkApiFetch(url).json() - it('answers every endpoint it advertises', () => { - const unanswered = listBenchmarkMcpTools() - .map((tool) => - `/api${tool.path.replace('{workspace}', WORKSPACE)}`.replace(/\{[^}]+\}/g, 'x') - ) - .filter((url) => !hasBenchmarkApiHandler(url)) - - // The draft-covered entries are refused by name before any fetch, so they are - // advertised without a handler on purpose. - expect(unanswered).toEqual([ - `/api/w/${WORKSPACE}/scripts/get/p/x`, - `/api/w/${WORKSPACE}/flows/create`, - `/api/w/${WORKSPACE}/schedules/delete/x`, - `/api/w/${WORKSPACE}/variables/get/x` - ]) - }) - - it('runs a deployed script by path, the way call_api_endpoint reaches it', async () => { - registerBenchmarkWorkspaceRunnables(WORKSPACE, { - scripts: [ - { - path: 'f/evals/greet', - summary: 'Greet', - language: 'bun', - content: 'export async function main() {}' - } - ] - }) - - const res = handleBenchmarkApiFetch( - `/api/w/${WORKSPACE}/jobs/run/p/${encodeURIComponent('f/evals/greet')}`, - { method: 'POST', body: JSON.stringify({ name: 'ada' }) } - ) - - expect(res.status).toBe(200) - const job = getBenchmarkCompletedJob(WORKSPACE, (await res.text()).trim()) - expect(job).toMatchObject({ success: true, args: { name: 'ada' } }) - }) - - it('serves a recorded job so a model can check the run it just started', async () => { - const id = createBenchmarkCompletedJob({ - workspace: WORKSPACE, - jobKind: 'preview', - result: 'Hello, World!' - }) - - const res = handleBenchmarkApiFetch(`/api/w/${WORKSPACE}/jobs_u/get/${id}`) - - expect(res.status).toBe(200) - expect(await res.json()).toMatchObject({ - id, - success: true, - result: 'Hello, World!' - }) - }) - - it('404s an unknown job id instead of letting the fetch fall through', () => { - expect(hasBenchmarkApiHandler(`/api/w/${WORKSPACE}/jobs_u/get/missing`)).toBe(true) - expect(handleBenchmarkApiFetch(`/api/w/${WORKSPACE}/jobs_u/get/missing`).status).toBe(404) + // The tool counts the list and maps over it, so it needs a bare array, not an envelope. + expect(Array.isArray(body)).toBe(true) + expect(body[0]).toMatchObject({ worker: expect.any(String), worker_group: expect.any(String) }) }) }) diff --git a/ai_evals/adapters/frontend/vitestAdapter.test.ts b/ai_evals/adapters/frontend/vitestAdapter.test.ts index c9e889b9a0..0c8a22a987 100644 --- a/ai_evals/adapters/frontend/vitestAdapter.test.ts +++ b/ai_evals/adapters/frontend/vitestAdapter.test.ts @@ -5,8 +5,8 @@ import { mkdir, writeFile } from 'fs/promises' import { dirname, resolve } from 'path' import { handleBenchmarkApiFetch, hasBenchmarkApiHandler } from './mockBackend' -// The API catalog executor issues relative fetch('/api/...') calls, which have -// no meaning in the vitest environment — serve the ones the benchmark handles. +// Some tools reach the backend by relative fetch('/api/...'), which has no meaning +// in the vitest environment — serve the ones the benchmark handles. // Every other relative fetch keeps its normal behavior (it fails the same way // it does without this stub) so unrelated tools see an unchanged environment. // The frontend builds API URLs from location.origin (fetchAvailableModels does), and node has no @@ -92,8 +92,7 @@ vi.mock('$lib/gen', async () => { runBenchmarkFlowPreview, runBenchmarkScriptByPath, runBenchmarkScriptPreview, - updateBenchmarkDraft, - listBenchmarkMcpTools + updateBenchmarkDraft } = await import('./mockBackend') function wrapService(target: T, overrides: Record): T { @@ -434,12 +433,6 @@ vi.mock('$lib/gen', async () => { queryResourceTypes: async (data: { workspace: string }) => hasBenchmarkWorkspace(data.workspace) ? [] : actual.ResourceService.queryResourceTypes(data) }), - McpService: wrapService(actual.McpService, { - listMcpTools: async (data: { workspace: string }) => - hasBenchmarkWorkspace(data.workspace) - ? listBenchmarkMcpTools() - : actual.McpService.listMcpTools(data) - }), VariableService: wrapService(actual.VariableService, { existsVariable: async (data: { workspace: string; path: string }) => hasBenchmarkWorkspace(data.workspace) diff --git a/ai_evals/cases/global.yaml b/ai_evals/cases/global.yaml index 7375aa09c0..f1388c3175 100644 --- a/ai_evals/cases/global.yaml +++ b/ai_evals/cases/global.yaml @@ -1919,11 +1919,10 @@ - when the lookup fails, tells the user instead of inventing table names - does not write scripts or resources to answer a read-only question -# --- Dedicated tools preferred over the API catalog --- -# The harness serves worker/queue reads itself (benchmark fetch handlers in -# adapters/frontend), so these cases do not require an mcp-enabled eval backend. -# The stale `api-catalog` in the id below is kept so results stay comparable -# across benchmark runs. +# --- Dedicated tools for workspace, run and instance state --- +# The harness serves the worker read itself (a benchmark fetch handler in +# adapters/frontend). The stale `api-catalog` in the id below is kept so +# results stay comparable across benchmark runs. - id: global-test30-api-catalog-workers prompt: |- @@ -1937,8 +1936,6 @@ requiredToolsUsed: - list_workers forbiddenToolsUsed: - - call_api_get - - call_api_endpoint - write_script - deploy_workspace_item # Read-only workspace inspection produces no draft; validate via tool use. @@ -1990,13 +1987,11 @@ requiredToolsUsed: - test_run_script forbiddenToolsUsed: - - call_api_endpoint - deploy_workspace_item - delete_workspace_item # The judge only sees the drafts artifact and cannot observe runs, so it always # docks the prompt's "run it" requirement — validate deterministically instead: - # draft content via valueIncludes, the test run via toolExpect (test_run_script - # required, call_api_endpoint forbidden). + # draft content via valueIncludes, the test run via toolExpect. skipJudge: true judgeChecklist: - creates an AI draft of f/evals/global/format_greeting with the name uppercased in the greeting @@ -2014,13 +2009,11 @@ requiredToolsUsed: - delete_workspace_item forbiddenToolsUsed: - - call_api_endpoint - - search_api_endpoints - write_script # Deletion produces no draft; validate via tool use. skipJudge: true judgeChecklist: - - deletes the deployed script via delete_workspace_item rather than a raw API endpoint + - deletes the deployed script via delete_workspace_item - id: global-test33-run-deployed-script-with-form prompt: |- @@ -2041,7 +2034,6 @@ - read_workspace_item forbiddenToolsUsed: - test_run_script - - call_api_endpoint - write_script - deploy_workspace_item # An empty form pushes the work back onto the user, so the prefill is part of @@ -2054,7 +2046,7 @@ # Running produces no draft, and the judge cannot observe runs; validate via tool use. skipJudge: true judgeChecklist: - - runs the deployed script through run_script rather than a preview test run or a raw API endpoint + - runs the deployed script through run_script rather than a preview test run - passes the name "ada" so the confirmation form comes up prefilled - id: global-test34-run-with-secret-from-variable @@ -2113,7 +2105,6 @@ - read_workspace_item forbiddenToolsUsed: - test_run_flow - - call_api_endpoint - write_flow - deploy_workspace_item # An empty form pushes the work back onto the user, so the prefill is part of @@ -2126,7 +2117,7 @@ # Running produces no draft, and the judge cannot observe runs; validate via tool use. skipJudge: true judgeChecklist: - - runs the deployed flow through run_flow rather than a preview test run or a raw API endpoint + - runs the deployed flow through run_flow rather than a preview test run - passes the customer "acme" so the confirmation form comes up prefilled - id: global-test36-draft-flow-test-run-not-deployed @@ -2151,7 +2142,6 @@ # version instead — the edit would not be in what ran. forbiddenToolsUsed: - run_flow - - call_api_endpoint - deploy_workspace_item # The judge cannot observe runs, and the edit's content is already pinned by # global-test5 on this fixture; what this case guards is where the run went. diff --git a/backend/.sqlx/query-044310c0e7745867a24a9d3653ff29c87ce48ffd89fac070c8314556f884ce55.json b/backend/.sqlx/query-044310c0e7745867a24a9d3653ff29c87ce48ffd89fac070c8314556f884ce55.json new file mode 100644 index 0000000000..be7ac83a7b --- /dev/null +++ b/backend/.sqlx/query-044310c0e7745867a24a9d3653ff29c87ce48ffd89fac070c8314556f884ce55.json @@ -0,0 +1,43 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT s.hash as hash, dm.deployment_msg as deployment_msg, s.created_at as created_at, s.created_by as created_by\n FROM script s LEFT JOIN deployment_metadata dm ON s.hash = dm.script_hash\n WHERE s.workspace_id = $1 AND s.path = $2\n ORDER by s.created_at DESC\n LIMIT $3 OFFSET $4", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "hash", + "type_info": "Int8" + }, + { + "ordinal": 1, + "name": "deployment_msg", + "type_info": "Text" + }, + { + "ordinal": 2, + "name": "created_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 3, + "name": "created_by", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Int8", + "Int8" + ] + }, + "nullable": [ + false, + true, + false, + false + ] + }, + "hash": "044310c0e7745867a24a9d3653ff29c87ce48ffd89fac070c8314556f884ce55" +} diff --git a/backend/.sqlx/query-09e3f8fa8a9e64048971983584bfa00ab137c70b73f381452cadfae6e1cda934.json b/backend/.sqlx/query-09e3f8fa8a9e64048971983584bfa00ab137c70b73f381452cadfae6e1cda934.json new file mode 100644 index 0000000000..a4d174992e --- /dev/null +++ b/backend/.sqlx/query-09e3f8fa8a9e64048971983584bfa00ab137c70b73f381452cadfae6e1cda934.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE draft AS d\n SET path = $3::text,\n value = (\n SELECT CASE\n WHEN s.clean -> 'path' = to_jsonb($2::text)\n OR s.clean -> 'draft_path' = to_jsonb($2::text)\n THEN to_json(\n s.clean\n || CASE WHEN s.clean -> 'path' = to_jsonb($2::text)\n THEN jsonb_build_object('path', $3::text)\n ELSE '{}'::jsonb END\n || CASE WHEN s.clean -> 'draft_path' = to_jsonb($2::text)\n THEN jsonb_build_object('draft_path', $3::text)\n ELSE '{}'::jsonb END\n )\n ELSE d.value\n END\n FROM (SELECT replace(replace(replace(d.value::text, chr(92) || chr(92), chr(1)),\n chr(92) || 'u0000', ''), chr(1), chr(92) || chr(92))::jsonb AS clean) s\n )\n WHERE d.workspace_id = $1\n AND d.path = $2::text\n AND d.typ::text = ANY($4::text[])", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Text", + "TextArray" + ] + }, + "nullable": [] + }, + "hash": "09e3f8fa8a9e64048971983584bfa00ab137c70b73f381452cadfae6e1cda934" +} diff --git a/backend/.sqlx/query-0cc6770a81ecaecafe0d9b7100f94b329c4ec7978b016af4c964415e0c38396c.json b/backend/.sqlx/query-0cc6770a81ecaecafe0d9b7100f94b329c4ec7978b016af4c964415e0c38396c.json new file mode 100644 index 0000000000..23d3b51c76 --- /dev/null +++ b/backend/.sqlx/query-0cc6770a81ecaecafe0d9b7100f94b329c4ec7978b016af4c964415e0c38396c.json @@ -0,0 +1,71 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT value as \"value!: sqlx::types::Json>\",\n created_at, base\n FROM draft\n WHERE workspace_id = $1\n AND (email = $2 OR email IS NULL)\n AND path = $3\n AND typ = $4\n ORDER BY email NULLS LAST\n LIMIT 1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "value!: sqlx::types::Json>", + "type_info": "Json" + }, + { + "ordinal": 1, + "name": "created_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 2, + "name": "base", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Text", + { + "Custom": { + "name": "draft_kind", + "kind": { + "Enum": [ + "script", + "flow", + "app", + "raw_app", + "resource", + "variable", + "trigger_schedule", + "trigger_webhook", + "trigger_default_email", + "trigger_email", + "trigger_http", + "trigger_websocket", + "trigger_postgres", + "trigger_kafka", + "trigger_nats", + "trigger_mqtt", + "trigger_sqs", + "trigger_gcp", + "trigger_azure", + "trigger_poll", + "trigger_cli", + "trigger_nextcloud", + "trigger_google", + "trigger_github", + "data_pipeline", + "trigger_amqp" + ] + } + } + } + ] + }, + "nullable": [ + false, + false, + true + ] + }, + "hash": "0cc6770a81ecaecafe0d9b7100f94b329c4ec7978b016af4c964415e0c38396c" +} diff --git a/backend/.sqlx/query-0d9a83d77632bc0de19a48eddb7e6f80b2a41db4f1d352920d0c295ea3a018b1.json b/backend/.sqlx/query-0d9a83d77632bc0de19a48eddb7e6f80b2a41db4f1d352920d0c295ea3a018b1.json new file mode 100644 index 0000000000..93251596cf --- /dev/null +++ b/backend/.sqlx/query-0d9a83d77632bc0de19a48eddb7e6f80b2a41db4f1d352920d0c295ea3a018b1.json @@ -0,0 +1,64 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE draft\n SET path = $3,\n -- Both path keys, not just the typed one: the editors mirror the\n -- typed path into the other while it differs from the row's path,\n -- and the loaders prefer the mirror — left naming the old location\n -- it un-does this move on the next save. `create_missing = false`\n -- on both, so a draft carrying only one keeps only one.\n value = to_json(\n jsonb_set(\n jsonb_set(\n CASE WHEN $7::text IS NULL THEN to_jsonb(value)\n ELSE jsonb_set(to_jsonb(value), ARRAY['summary'], to_jsonb($7::text))\n END,\n ARRAY[$5::text], to_jsonb($3::text), false\n ),\n ARRAY[$8::text], to_jsonb($3::text), false\n )\n )\n WHERE workspace_id = $1\n AND path = $2\n AND typ = $4\n AND email = $6\n -- A pre-sanitizer NUL escape makes `to_jsonb` raise 22P05. Excluded\n -- here so the statement can't 500; reported below instead. Unlike the\n -- passive carry, rewriting the value IS this operation, so skipping it\n -- silently would move the row and leave its typed path stale.\n AND position(chr(92) || 'u0000' in replace(value::text, chr(92) || chr(92), '')) = 0\n -- Skipped on a summary-only edit, where the \"target\" row is this\n -- row and the guard would refuse the update against itself.\n AND ($2 = $3 OR NOT EXISTS (\n SELECT 1 FROM draft o\n WHERE o.workspace_id = $1 AND o.path = $3 AND o.typ::text = ANY($9::text[])\n -- Of this kind only the caller's own row and the legacy one collide:\n -- teammates' drafts of one item share its path by design, but a deploy\n -- there wipes those two together, so a second would discard edits the\n -- caller never saw. The other app kind is a different item on the same\n -- deployed path, so it collides whoever owns it.\n AND (o.typ <> $4 OR o.email = $6 OR o.email IS NULL)\n ))\n RETURNING id", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Text", + { + "Custom": { + "name": "draft_kind", + "kind": { + "Enum": [ + "script", + "flow", + "app", + "raw_app", + "resource", + "variable", + "trigger_schedule", + "trigger_webhook", + "trigger_default_email", + "trigger_email", + "trigger_http", + "trigger_websocket", + "trigger_postgres", + "trigger_kafka", + "trigger_nats", + "trigger_mqtt", + "trigger_sqs", + "trigger_gcp", + "trigger_azure", + "trigger_poll", + "trigger_cli", + "trigger_nextcloud", + "trigger_google", + "trigger_github", + "data_pipeline", + "trigger_amqp" + ] + } + } + }, + "Text", + "Text", + "Text", + "Text", + "TextArray" + ] + }, + "nullable": [ + false + ] + }, + "hash": "0d9a83d77632bc0de19a48eddb7e6f80b2a41db4f1d352920d0c295ea3a018b1" +} diff --git a/backend/.sqlx/query-2b93351b9c2c91272c3f69c9c8eba192d2e4d087c7d7db8d3b257bb864c8265d.json b/backend/.sqlx/query-0f374865faa5bdf884345808a004ce891a85daa00e037aba4b65b60f87243bd3.json similarity index 65% rename from backend/.sqlx/query-2b93351b9c2c91272c3f69c9c8eba192d2e4d087c7d7db8d3b257bb864c8265d.json rename to backend/.sqlx/query-0f374865faa5bdf884345808a004ce891a85daa00e037aba4b65b60f87243bd3.json index b82fcf2e17..2ff7820d86 100644 --- a/backend/.sqlx/query-2b93351b9c2c91272c3f69c9c8eba192d2e4d087c7d7db8d3b257bb864c8265d.json +++ b/backend/.sqlx/query-0f374865faa5bdf884345808a004ce891a85daa00e037aba4b65b60f87243bd3.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "INSERT INTO datatable_migrations (workspace_id, datatable, timestamp, name, code_up, code_down) VALUES ($1, $2, $3, $4, $5, $6) ON CONFLICT (workspace_id, datatable, timestamp) DO UPDATE SET name = EXCLUDED.name, code_up = EXCLUDED.code_up, code_down = EXCLUDED.code_down WHERE NOT $7 OR (datatable_migrations.name = $8::text AND datatable_migrations.code_up = $9::text AND datatable_migrations.code_down IS NOT DISTINCT FROM $10::text)", + "query": "INSERT INTO datatable_migrations (workspace_id, datatable, timestamp, name, code_up, code_down) VALUES ($1, $2, $3, $4, $5, $6) ON CONFLICT (workspace_id, datatable, timestamp) DO UPDATE SET name = EXCLUDED.name, code_up = EXCLUDED.code_up, code_down = EXCLUDED.code_down WHERE datatable_migrations.name = $7::text AND datatable_migrations.code_up = $8::text AND datatable_migrations.code_down IS NOT DISTINCT FROM $9::text", "describe": { "columns": [], "parameters": { @@ -11,7 +11,6 @@ "Varchar", "Text", "Text", - "Bool", "Text", "Text", "Text" @@ -19,5 +18,5 @@ }, "nullable": [] }, - "hash": "2b93351b9c2c91272c3f69c9c8eba192d2e4d087c7d7db8d3b257bb864c8265d" + "hash": "0f374865faa5bdf884345808a004ce891a85daa00e037aba4b65b60f87243bd3" } diff --git a/backend/.sqlx/query-11af34e5fe5b8ed59453d58628ed5fb3b2120835158f22d715b73460f6a55108.json b/backend/.sqlx/query-11af34e5fe5b8ed59453d58628ed5fb3b2120835158f22d715b73460f6a55108.json new file mode 100644 index 0000000000..fdf0049344 --- /dev/null +++ b/backend/.sqlx/query-11af34e5fe5b8ed59453d58628ed5fb3b2120835158f22d715b73460f6a55108.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM draft_move WHERE email = $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [] + }, + "hash": "11af34e5fe5b8ed59453d58628ed5fb3b2120835158f22d715b73460f6a55108" +} diff --git a/backend/.sqlx/query-132bab72036874c33805ff194fce6641817d09528f9d1b7210abe9881b817710.json b/backend/.sqlx/query-132bab72036874c33805ff194fce6641817d09528f9d1b7210abe9881b817710.json new file mode 100644 index 0000000000..15d1703e5d --- /dev/null +++ b/backend/.sqlx/query-132bab72036874c33805ff194fce6641817d09528f9d1b7210abe9881b817710.json @@ -0,0 +1,32 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO remote_deploy_token\n (workspace_id, email, base_url, remote_workspace_id, token, remote_email, proxy_key,\n connected_at, member_since, target_changed_at)\n SELECT $1, $2, $3, $4, $5, $6, $7, $8, $9, $10\n WHERE $9::timestamptz IS NOT NULL\n OR EXISTS (SELECT 1 FROM token WHERE token_hash = $11)\n ON CONFLICT (workspace_id, email) DO UPDATE SET\n base_url = EXCLUDED.base_url, remote_workspace_id = EXCLUDED.remote_workspace_id,\n token = EXCLUDED.token, remote_email = EXCLUDED.remote_email,\n proxy_key = EXCLUDED.proxy_key, connected_at = EXCLUDED.connected_at,\n member_since = EXCLUDED.member_since, target_changed_at = EXCLUDED.target_changed_at\n WHERE remote_deploy_token.connected_at < EXCLUDED.connected_at\n RETURNING connected_at", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "connected_at", + "type_info": "Timestamptz" + } + ], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Varchar", + "Varchar", + "Text", + "Varchar", + "Varchar", + "Timestamptz", + "Timestamptz", + "Timestamptz", + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "132bab72036874c33805ff194fce6641817d09528f9d1b7210abe9881b817710" +} diff --git a/backend/.sqlx/query-9a1483a81f5b086e0765d3d69483e29b09f66090e1f9d394564c16d921d2e66c.json b/backend/.sqlx/query-148fbee57c6abbb6f361818c5b6bfc8a1db451d31dfddbb5be945b3551c8becb.json similarity index 57% rename from backend/.sqlx/query-9a1483a81f5b086e0765d3d69483e29b09f66090e1f9d394564c16d921d2e66c.json rename to backend/.sqlx/query-148fbee57c6abbb6f361818c5b6bfc8a1db451d31dfddbb5be945b3551c8becb.json index e751b8fc7c..09e5b51ab9 100644 --- a/backend/.sqlx/query-9a1483a81f5b086e0765d3d69483e29b09f66090e1f9d394564c16d921d2e66c.json +++ b/backend/.sqlx/query-148fbee57c6abbb6f361818c5b6bfc8a1db451d31dfddbb5be945b3551c8becb.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT s.hash as hash, dm.deployment_msg as deployment_msg, s.created_at as created_at\n FROM script s LEFT JOIN deployment_metadata dm ON s.hash = dm.script_hash\n WHERE s.workspace_id = $1 AND s.path = $2\n ORDER by s.created_at DESC", + "query": "SELECT s.hash as hash, dm.deployment_msg as deployment_msg, s.created_at as created_at, s.created_by as created_by\n FROM script s LEFT JOIN deployment_metadata dm ON s.hash = dm.script_hash\n WHERE s.workspace_id = $1 AND s.path = $2\n ORDER by s.created_at DESC LIMIT 1", "describe": { "columns": [ { @@ -17,6 +17,11 @@ "ordinal": 2, "name": "created_at", "type_info": "Timestamptz" + }, + { + "ordinal": 3, + "name": "created_by", + "type_info": "Varchar" } ], "parameters": { @@ -28,8 +33,9 @@ "nullable": [ false, true, + false, false ] }, - "hash": "9a1483a81f5b086e0765d3d69483e29b09f66090e1f9d394564c16d921d2e66c" + "hash": "148fbee57c6abbb6f361818c5b6bfc8a1db451d31dfddbb5be945b3551c8becb" } diff --git a/backend/.sqlx/query-14cef81d324504d9cc90a9ffeb9f73daf9e09cac11685988bc861cc348a0c518.json b/backend/.sqlx/query-14cef81d324504d9cc90a9ffeb9f73daf9e09cac11685988bc861cc348a0c518.json new file mode 100644 index 0000000000..45fbabdae9 --- /dev/null +++ b/backend/.sqlx/query-14cef81d324504d9cc90a9ffeb9f73daf9e09cac11685988bc861cc348a0c518.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT remote_deploy_target FROM workspace_settings WHERE workspace_id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "remote_deploy_target", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + true + ] + }, + "hash": "14cef81d324504d9cc90a9ffeb9f73daf9e09cac11685988bc861cc348a0c518" +} diff --git a/backend/.sqlx/query-18b38efcb141101192b372e7d7e7e47481877f3ab717682f3b84e665c95a5214.json b/backend/.sqlx/query-18b38efcb141101192b372e7d7e7e47481877f3ab717682f3b84e665c95a5214.json new file mode 100644 index 0000000000..d26cdf3a7d --- /dev/null +++ b/backend/.sqlx/query-18b38efcb141101192b372e7d7e7e47481877f3ab717682f3b84e665c95a5214.json @@ -0,0 +1,49 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT a.id as app_id, av.id as version_id, dm.deployment_msg as deployment_msg,\n av.created_by as created_by, av.created_at as created_at\n FROM app a\n JOIN LATERAL (\n SELECT av2.id, COALESCE(v.ord, 0) AS ord\n FROM app_version av2\n LEFT JOIN unnest(a.versions) WITH ORDINALITY AS v(id, ord) ON v.id = av2.id\n WHERE av2.app_id = a.id\n ORDER BY ord DESC, av2.id DESC\n LIMIT $3 OFFSET $4\n ) page ON TRUE\n JOIN app_version av ON av.id = page.id\n LEFT JOIN deployment_metadata dm ON av.id = dm.app_version\n WHERE a.workspace_id = $1 AND a.path = $2\n ORDER BY page.ord DESC", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "app_id", + "type_info": "Int8" + }, + { + "ordinal": 1, + "name": "version_id", + "type_info": "Int8" + }, + { + "ordinal": 2, + "name": "deployment_msg", + "type_info": "Text" + }, + { + "ordinal": 3, + "name": "created_by", + "type_info": "Varchar" + }, + { + "ordinal": 4, + "name": "created_at", + "type_info": "Timestamptz" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Int8", + "Int8" + ] + }, + "nullable": [ + false, + false, + true, + false, + false + ] + }, + "hash": "18b38efcb141101192b372e7d7e7e47481877f3ab717682f3b84e665c95a5214" +} diff --git a/backend/.sqlx/query-46f00a75b2e7e4ac70758a9687070f68bc0421f1aa228f80157adda63191d33b.json b/backend/.sqlx/query-1c6859291963a9ca38d73f500213a575bfcd4469524f0dc8812470737759f02b.json similarity index 74% rename from backend/.sqlx/query-46f00a75b2e7e4ac70758a9687070f68bc0421f1aa228f80157adda63191d33b.json rename to backend/.sqlx/query-1c6859291963a9ca38d73f500213a575bfcd4469524f0dc8812470737759f02b.json index dcd3573e92..7b78a7d9b4 100644 --- a/backend/.sqlx/query-46f00a75b2e7e4ac70758a9687070f68bc0421f1aa228f80157adda63191d33b.json +++ b/backend/.sqlx/query-1c6859291963a9ca38d73f500213a575bfcd4469524f0dc8812470737759f02b.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "WITH legacy AS (\n DELETE FROM draft\n WHERE workspace_id = $1 AND path = $2 AND typ = $3 AND email IS NULL\n RETURNING value\n )\n INSERT INTO draft (workspace_id, email, path, typ, value, created_at)\n SELECT $1, $4, $2, $3, value, now() FROM legacy\n ON CONFLICT (workspace_id, path, typ, email) WHERE email IS NOT NULL\n DO UPDATE SET value = EXCLUDED.value, created_at = now()\n RETURNING 1 as \"one!\"", + "query": "WITH legacy AS (\n DELETE FROM draft\n WHERE workspace_id = $1 AND path = $2 AND typ = $3 AND email IS NULL\n RETURNING value, base\n )\n INSERT INTO draft (workspace_id, email, path, typ, value, created_at, base)\n SELECT $1, $4, $2, $3, value, now(), base FROM legacy\n ON CONFLICT (workspace_id, path, typ, email) WHERE email IS NOT NULL\n DO UPDATE SET value = EXCLUDED.value, created_at = now(), base = EXCLUDED.base\n RETURNING 1 as \"one!\"", "describe": { "columns": [ { @@ -55,5 +55,5 @@ null ] }, - "hash": "46f00a75b2e7e4ac70758a9687070f68bc0421f1aa228f80157adda63191d33b" + "hash": "1c6859291963a9ca38d73f500213a575bfcd4469524f0dc8812470737759f02b" } diff --git a/backend/.sqlx/query-207db6c65949b85fda6d6289921bb04695b1c50e17755dad3bd162254f67458f.json b/backend/.sqlx/query-207db6c65949b85fda6d6289921bb04695b1c50e17755dad3bd162254f67458f.json new file mode 100644 index 0000000000..e698dbecb2 --- /dev/null +++ b/backend/.sqlx/query-207db6c65949b85fda6d6289921bb04695b1c50e17755dad3bd162254f67458f.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE remote_deploy_token SET token = $1\n WHERE workspace_id = $2 AND email = $3", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "207db6c65949b85fda6d6289921bb04695b1c50e17755dad3bd162254f67458f" +} diff --git a/backend/.sqlx/query-232ff36c2d522383061cafbf9639698dbc44a18c0f016aae94a1705f06fd1e48.json b/backend/.sqlx/query-232ff36c2d522383061cafbf9639698dbc44a18c0f016aae94a1705f06fd1e48.json new file mode 100644 index 0000000000..f32e47b387 --- /dev/null +++ b/backend/.sqlx/query-232ff36c2d522383061cafbf9639698dbc44a18c0f016aae94a1705f06fd1e48.json @@ -0,0 +1,18 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO draft_move (workspace_id, typ, old_path, new_path, email)\n SELECT m.workspace_id, m.typ, m.old_path, $4, $5::text\n FROM draft_move m\n WHERE m.workspace_id = $1 AND m.typ::text = ANY($2::text[])\n AND m.new_path = $3 AND m.email IS NULL\n AND NOT EXISTS (\n SELECT 1 FROM draft_move o\n WHERE o.workspace_id = m.workspace_id AND o.typ = m.typ\n AND o.old_path = m.old_path AND o.email = $5::text\n )", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "TextArray", + "Text", + "Varchar", + "Text" + ] + }, + "nullable": [] + }, + "hash": "232ff36c2d522383061cafbf9639698dbc44a18c0f016aae94a1705f06fd1e48" +} diff --git a/backend/.sqlx/query-359cd29f531d263a8cf7205e0869229a610767087f01e0154be8da0620fa114b.json b/backend/.sqlx/query-359cd29f531d263a8cf7205e0869229a610767087f01e0154be8da0620fa114b.json deleted file mode 100644 index 23cf739f90..0000000000 --- a/backend/.sqlx/query-359cd29f531d263a8cf7205e0869229a610767087f01e0154be8da0620fa114b.json +++ /dev/null @@ -1,38 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "WITH all_audit AS (SELECT username, operation, timestamp FROM audit_partitioned UNION ALL SELECT username, operation, timestamp FROM audit),\n active_users as (SELECT distinct username as email FROM all_audit WHERE timestamp > NOW() - INTERVAL '1 month' AND (operation = 'users.login' OR operation = 'oauth.login' OR operation = 'users.token.refresh')),\n active_authors as (SELECT distinct email FROM usr WHERE usr.operator IS false AND email IN (SELECT email FROM active_users)),\n active_authors_agg as (SELECT array_agg(email) as authors FROM active_authors),\n active_ops_agg as (SELECT array_agg(email) as operators from active_users WHERE email NOT IN (SELECT email FROM active_authors))\n SELECT active_authors_agg.authors, active_ops_agg.operators, array_length(active_authors_agg.authors, 1) as author_count, array_length(active_ops_agg.operators, 1) as operator_count FROM active_authors_agg, active_ops_agg", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "authors", - "type_info": "VarcharArray" - }, - { - "ordinal": 1, - "name": "operators", - "type_info": "VarcharArray" - }, - { - "ordinal": 2, - "name": "author_count", - "type_info": "Int4" - }, - { - "ordinal": 3, - "name": "operator_count", - "type_info": "Int4" - } - ], - "parameters": { - "Left": [] - }, - "nullable": [ - null, - null, - null, - null - ] - }, - "hash": "359cd29f531d263a8cf7205e0869229a610767087f01e0154be8da0620fa114b" -} diff --git a/backend/.sqlx/query-3a015ab279785a15f89c89d8c4d155e85370d4c2b7ed14fbb510919e7624ca1c.json b/backend/.sqlx/query-3a015ab279785a15f89c89d8c4d155e85370d4c2b7ed14fbb510919e7624ca1c.json new file mode 100644 index 0000000000..c4cc310dbf --- /dev/null +++ b/backend/.sqlx/query-3a015ab279785a15f89c89d8c4d155e85370d4c2b7ed14fbb510919e7624ca1c.json @@ -0,0 +1,18 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO draft_move (workspace_id, typ, old_path, new_path, email)\n SELECT $1, t::draft_kind, $3, $4, $5 FROM unnest($2::text[]) t", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "TextArray", + "Varchar", + "Varchar", + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "3a015ab279785a15f89c89d8c4d155e85370d4c2b7ed14fbb510919e7624ca1c" +} diff --git a/backend/.sqlx/query-3a0ac41e645225bbe54b83f246500ca014154e76eb322646e7e4e753b6420acd.json b/backend/.sqlx/query-3a0ac41e645225bbe54b83f246500ca014154e76eb322646e7e4e753b6420acd.json new file mode 100644 index 0000000000..b3b683eca1 --- /dev/null +++ b/backend/.sqlx/query-3a0ac41e645225bbe54b83f246500ca014154e76eb322646e7e4e753b6420acd.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM token WHERE token_prefix = 'GONE_TOKEN'", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "3a0ac41e645225bbe54b83f246500ca014154e76eb322646e7e4e753b6420acd" +} diff --git a/backend/.sqlx/query-3b6f1e9765de11ef2da0ec5388c72e4a01f50d48b76b90746eef72919c408e44.json b/backend/.sqlx/query-3b6f1e9765de11ef2da0ec5388c72e4a01f50d48b76b90746eef72919c408e44.json new file mode 100644 index 0000000000..9901e61b6a --- /dev/null +++ b/backend/.sqlx/query-3b6f1e9765de11ef2da0ec5388c72e4a01f50d48b76b90746eef72919c408e44.json @@ -0,0 +1,28 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT email, token FROM remote_deploy_token\n WHERE workspace_id = $1 AND token <> '' FOR UPDATE", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "email", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "token", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false, + false + ] + }, + "hash": "3b6f1e9765de11ef2da0ec5388c72e4a01f50d48b76b90746eef72919c408e44" +} diff --git a/backend/.sqlx/query-3db5f1d8ec1f8d867cf01c066876122e78f0c76a93a1d9860e8dff05d9a02c09.json b/backend/.sqlx/query-3db5f1d8ec1f8d867cf01c066876122e78f0c76a93a1d9860e8dff05d9a02c09.json new file mode 100644 index 0000000000..db2181ae3c --- /dev/null +++ b/backend/.sqlx/query-3db5f1d8ec1f8d867cf01c066876122e78f0c76a93a1d9860e8dff05d9a02c09.json @@ -0,0 +1,18 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE draft_move SET new_path = $4\n WHERE workspace_id = $1 AND typ::text = ANY($2::text[])\n AND new_path = $3 AND ($5::text IS NULL OR email = $5)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "TextArray", + "Text", + "Varchar", + "Text" + ] + }, + "nullable": [] + }, + "hash": "3db5f1d8ec1f8d867cf01c066876122e78f0c76a93a1d9860e8dff05d9a02c09" +} diff --git a/backend/.sqlx/query-51e1741eb9e959945302fd6f7e8c621b9b03b92f6c20ab1bdd1637232f26952d.json b/backend/.sqlx/query-51e1741eb9e959945302fd6f7e8c621b9b03b92f6c20ab1bdd1637232f26952d.json new file mode 100644 index 0000000000..b6babc98b0 --- /dev/null +++ b/backend/.sqlx/query-51e1741eb9e959945302fd6f7e8c621b9b03b92f6c20ab1bdd1637232f26952d.json @@ -0,0 +1,41 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT clock_timestamp() AS \"started_at!\", s.remote_deploy_target,\n s.remote_deploy_target_changed_at,\n (SELECT u.created_at FROM usr u\n WHERE u.workspace_id = s.workspace_id AND u.email = $2) AS member_since\n FROM workspace_settings s WHERE s.workspace_id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "started_at!", + "type_info": "Timestamptz" + }, + { + "ordinal": 1, + "name": "remote_deploy_target", + "type_info": "Jsonb" + }, + { + "ordinal": 2, + "name": "remote_deploy_target_changed_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 3, + "name": "member_since", + "type_info": "Timestamptz" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + null, + true, + true, + null + ] + }, + "hash": "51e1741eb9e959945302fd6f7e8c621b9b03b92f6c20ab1bdd1637232f26952d" +} diff --git a/backend/.sqlx/query-530fe81babd82e7b4a9142962c62750560e459acc7c83c48d47d7bdad29051be.json b/backend/.sqlx/query-530fe81babd82e7b4a9142962c62750560e459acc7c83c48d47d7bdad29051be.json new file mode 100644 index 0000000000..65266cbae0 --- /dev/null +++ b/backend/.sqlx/query-530fe81babd82e7b4a9142962c62750560e459acc7c83c48d47d7bdad29051be.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO token (token_hash, token_prefix, token, email, label, super_admin, read_only)\n VALUES (encode(sha256('READ_ONLY_TOKEN'::bytea), 'hex'), 'READ_ONLY_', 'READ_ONLY_TOKEN',\n 'test@windmill.dev', 'read only', false, true)", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "530fe81babd82e7b4a9142962c62750560e459acc7c83c48d47d7bdad29051be" +} diff --git a/backend/.sqlx/query-5517b714165f959a1df1a0ac9cbe848344974bc2271459ba4ae5952274c11429.json b/backend/.sqlx/query-5517b714165f959a1df1a0ac9cbe848344974bc2271459ba4ae5952274c11429.json new file mode 100644 index 0000000000..c29a42c56b --- /dev/null +++ b/backend/.sqlx/query-5517b714165f959a1df1a0ac9cbe848344974bc2271459ba4ae5952274c11429.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE workspace_settings SET remote_deploy_target_changed_at = '2000-01-01'\n WHERE workspace_id = 'test-workspace'", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "5517b714165f959a1df1a0ac9cbe848344974bc2271459ba4ae5952274c11429" +} diff --git a/backend/.sqlx/query-5959c60da91f1e62080a4455e2a4627cb1fd3d8c70210bf7f38cb222dd15d88f.json b/backend/.sqlx/query-5959c60da91f1e62080a4455e2a4627cb1fd3d8c70210bf7f38cb222dd15d88f.json new file mode 100644 index 0000000000..b78808a389 --- /dev/null +++ b/backend/.sqlx/query-5959c60da91f1e62080a4455e2a4627cb1fd3d8c70210bf7f38cb222dd15d88f.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM draft_move dest\n WHERE dest.email = $1\n AND EXISTS (SELECT 1 FROM draft_move src\n WHERE src.email = $2\n AND src.workspace_id = dest.workspace_id\n AND src.old_path = dest.old_path\n AND src.typ = dest.typ)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "5959c60da91f1e62080a4455e2a4627cb1fd3d8c70210bf7f38cb222dd15d88f" +} diff --git a/backend/.sqlx/query-5b292c49ce6b64f4de61f74e18831b3f107fc6d0312ac3da9247a922bcb05b67.json b/backend/.sqlx/query-5b292c49ce6b64f4de61f74e18831b3f107fc6d0312ac3da9247a922bcb05b67.json new file mode 100644 index 0000000000..7633c95854 --- /dev/null +++ b/backend/.sqlx/query-5b292c49ce6b64f4de61f74e18831b3f107fc6d0312ac3da9247a922bcb05b67.json @@ -0,0 +1,43 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT t.token, t.remote_email, t.proxy_key, t.connected_at FROM remote_deploy_token t\n JOIN workspace_settings s ON s.workspace_id = t.workspace_id\n WHERE t.workspace_id = $1 AND t.email = $2 AND t.base_url = $3\n AND t.remote_workspace_id = $4 AND t.token <> ''\n AND s.remote_deploy_target_changed_at IS NOT DISTINCT FROM t.target_changed_at\n AND (EXISTS (SELECT 1 FROM usr u WHERE u.workspace_id = t.workspace_id\n AND u.email = t.email AND u.created_at = t.member_since)\n OR EXISTS (SELECT 1 FROM password p WHERE p.email = t.email AND p.super_admin))", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "token", + "type_info": "Text" + }, + { + "ordinal": 1, + "name": "remote_email", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "proxy_key", + "type_info": "Varchar" + }, + { + "ordinal": 3, + "name": "connected_at", + "type_info": "Timestamptz" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Text", + "Text" + ] + }, + "nullable": [ + false, + false, + false, + false + ] + }, + "hash": "5b292c49ce6b64f4de61f74e18831b3f107fc6d0312ac3da9247a922bcb05b67" +} diff --git a/backend/.sqlx/query-6b58f2df413a867a44311fd779a2017c41b9a0e3985a25372e5f64723124e473.json b/backend/.sqlx/query-6b58f2df413a867a44311fd779a2017c41b9a0e3985a25372e5f64723124e473.json new file mode 100644 index 0000000000..20b5bd1a94 --- /dev/null +++ b/backend/.sqlx/query-6b58f2df413a867a44311fd779a2017c41b9a0e3985a25372e5f64723124e473.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO remote_deploy_token\n (workspace_id, email, base_url, remote_workspace_id, token, remote_email, proxy_key,\n connected_at)\n VALUES ($1, $2, '', '', '', '', '', clock_timestamp())\n ON CONFLICT (workspace_id, email) DO UPDATE SET\n token = '', proxy_key = '', connected_at = clock_timestamp()", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "6b58f2df413a867a44311fd779a2017c41b9a0e3985a25372e5f64723124e473" +} diff --git a/backend/.sqlx/query-6fcdb09cdd7fedd7e54fdc0e49203f453fc1b85272fe212c0e1bf0ebd28bf58a.json b/backend/.sqlx/query-6fcdb09cdd7fedd7e54fdc0e49203f453fc1b85272fe212c0e1bf0ebd28bf58a.json deleted file mode 100644 index 17fbb0d5bb..0000000000 --- a/backend/.sqlx/query-6fcdb09cdd7fedd7e54fdc0e49203f453fc1b85272fe212c0e1bf0ebd28bf58a.json +++ /dev/null @@ -1,38 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n WITH capped AS (\n SELECT timestamp, operation, resource, parameters\n FROM audit_partitioned\n WHERE workspace_id = 'admins'\n AND operation = 'workspace_fairness.capped'\n UNION ALL\n SELECT timestamp, operation, resource, parameters\n FROM audit\n WHERE workspace_id = 'admins'\n AND operation = 'workspace_fairness.capped'\n ORDER BY timestamp DESC\n LIMIT 200\n ), uncapped AS (\n SELECT timestamp, operation, resource, parameters\n FROM audit_partitioned\n WHERE workspace_id = 'admins'\n AND operation = 'workspace_fairness.uncapped'\n UNION ALL\n SELECT timestamp, operation, resource, parameters\n FROM audit\n WHERE workspace_id = 'admins'\n AND operation = 'workspace_fairness.uncapped'\n ORDER BY timestamp DESC\n LIMIT 200\n )\n SELECT timestamp AS \"timestamp!\",\n operation::text AS \"operation!\",\n resource AS workspace_id,\n parameters\n FROM (SELECT * FROM capped UNION ALL SELECT * FROM uncapped) e\n ORDER BY timestamp DESC\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "timestamp!", - "type_info": "Timestamptz" - }, - { - "ordinal": 1, - "name": "operation!", - "type_info": "Text" - }, - { - "ordinal": 2, - "name": "workspace_id", - "type_info": "Varchar" - }, - { - "ordinal": 3, - "name": "parameters", - "type_info": "Jsonb" - } - ], - "parameters": { - "Left": [] - }, - "nullable": [ - null, - null, - null, - null - ] - }, - "hash": "6fcdb09cdd7fedd7e54fdc0e49203f453fc1b85272fe212c0e1bf0ebd28bf58a" -} diff --git a/backend/.sqlx/query-7761959d05fff8c314d04c1bc9fb4e84a61008dbec0995ae192f1b3a1c261579.json b/backend/.sqlx/query-7761959d05fff8c314d04c1bc9fb4e84a61008dbec0995ae192f1b3a1c261579.json new file mode 100644 index 0000000000..c8b20a20ea --- /dev/null +++ b/backend/.sqlx/query-7761959d05fff8c314d04c1bc9fb4e84a61008dbec0995ae192f1b3a1c261579.json @@ -0,0 +1,72 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO draft (workspace_id, email, path, typ, value, created_at, base)\n VALUES ($1, $2, $3::text, $4,\n CASE WHEN $10::bool\n THEN to_json($5::text::jsonb || jsonb_strip_nulls(jsonb_build_object(\n 'path', CASE WHEN $5::text::jsonb -> 'path' = to_jsonb($11::text)\n THEN to_jsonb($3::text) END,\n 'draft_path', CASE WHEN $5::text::jsonb -> 'draft_path' = to_jsonb($11::text)\n THEN to_jsonb($3::text) END)))\n ELSE $5::text::json\n END,\n COALESCE($8::timestamptz, now()), $9)\n ON CONFLICT (workspace_id, path, typ, email) WHERE email IS NOT NULL\n DO UPDATE SET value = CASE\n WHEN NOT $10::bool\n OR position(chr(92) || 'u0000' in replace(draft.value::text, chr(92) || chr(92), '')) > 0\n THEN EXCLUDED.value\n ELSE to_json((to_jsonb(EXCLUDED.value) - 'path' - 'draft_path')\n || jsonb_strip_nulls(jsonb_build_object(\n 'path', to_jsonb(draft.value) -> 'path',\n 'draft_path', to_jsonb(draft.value) -> 'draft_path')))\n END,\n created_at = EXCLUDED.created_at,\n base = EXCLUDED.base\n WHERE $7::bool = true\n OR $6::timestamptz IS NULL\n OR draft.created_at <= $6::timestamptz\n RETURNING path, created_at", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "path", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "created_at", + "type_info": "Timestamptz" + } + ], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Text", + { + "Custom": { + "name": "draft_kind", + "kind": { + "Enum": [ + "script", + "flow", + "app", + "raw_app", + "resource", + "variable", + "trigger_schedule", + "trigger_webhook", + "trigger_default_email", + "trigger_email", + "trigger_http", + "trigger_websocket", + "trigger_postgres", + "trigger_kafka", + "trigger_nats", + "trigger_mqtt", + "trigger_sqs", + "trigger_gcp", + "trigger_azure", + "trigger_poll", + "trigger_cli", + "trigger_nextcloud", + "trigger_google", + "trigger_github", + "data_pipeline", + "trigger_amqp" + ] + } + } + }, + "Text", + "Timestamptz", + "Bool", + "Timestamptz", + "Text", + "Bool", + "Text" + ] + }, + "nullable": [ + false, + false + ] + }, + "hash": "7761959d05fff8c314d04c1bc9fb4e84a61008dbec0995ae192f1b3a1c261579" +} diff --git a/backend/.sqlx/query-797896e0152046844dbfee9d0e57e0055d742e44119bfb31ae7ff37a89b34d14.json b/backend/.sqlx/query-797896e0152046844dbfee9d0e57e0055d742e44119bfb31ae7ff37a89b34d14.json new file mode 100644 index 0000000000..ce3d615eb0 --- /dev/null +++ b/backend/.sqlx/query-797896e0152046844dbfee9d0e57e0055d742e44119bfb31ae7ff37a89b34d14.json @@ -0,0 +1,18 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM draft_move\n WHERE workspace_id = $1 AND typ::text = ANY($2::text[])\n AND old_path IN ($3, $4) AND ($5::text IS NULL OR email = $5)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "TextArray", + "Varchar", + "Varchar", + "Text" + ] + }, + "nullable": [] + }, + "hash": "797896e0152046844dbfee9d0e57e0055d742e44119bfb31ae7ff37a89b34d14" +} diff --git a/backend/.sqlx/query-7ff60d155510303e307c47dcb6153ccaac72871eb68822d8e296c0d747e55f35.json b/backend/.sqlx/query-7ff60d155510303e307c47dcb6153ccaac72871eb68822d8e296c0d747e55f35.json new file mode 100644 index 0000000000..6a004023a0 --- /dev/null +++ b/backend/.sqlx/query-7ff60d155510303e307c47dcb6153ccaac72871eb68822d8e296c0d747e55f35.json @@ -0,0 +1,43 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT flow_version.id, flow_version.created_at, flow_version.created_by, deployment_metadata.deployment_msg FROM flow_version\n LEFT JOIN deployment_metadata ON flow_version.id = deployment_metadata.flow_version\n WHERE flow_version.path = $1 AND flow_version.workspace_id = $2\n ORDER BY flow_version.created_at DESC\n LIMIT $3 OFFSET $4", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Int8" + }, + { + "ordinal": 1, + "name": "created_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 2, + "name": "created_by", + "type_info": "Varchar" + }, + { + "ordinal": 3, + "name": "deployment_msg", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Int8", + "Int8" + ] + }, + "nullable": [ + false, + false, + false, + true + ] + }, + "hash": "7ff60d155510303e307c47dcb6153ccaac72871eb68822d8e296c0d747e55f35" +} diff --git a/backend/.sqlx/query-81d5cceb2e405e1d17730dfa93a32f1bca7b235af9db581147d838454a4c9083.json b/backend/.sqlx/query-81d5cceb2e405e1d17730dfa93a32f1bca7b235af9db581147d838454a4c9083.json new file mode 100644 index 0000000000..40b6e22db6 --- /dev/null +++ b/backend/.sqlx/query-81d5cceb2e405e1d17730dfa93a32f1bca7b235af9db581147d838454a4c9083.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM remote_deploy_token WHERE workspace_id = $1\n AND ($2::text IS NULL OR base_url <> $2 OR remote_workspace_id <> $3)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "81d5cceb2e405e1d17730dfa93a32f1bca7b235af9db581147d838454a4c9083" +} diff --git a/backend/.sqlx/query-87afd5289e54b6accda8a399ec3c66df3d19217059f62903be3c74eadb9c15d5.json b/backend/.sqlx/query-87afd5289e54b6accda8a399ec3c66df3d19217059f62903be3c74eadb9c15d5.json deleted file mode 100644 index 8cdb2d5907..0000000000 --- a/backend/.sqlx/query-87afd5289e54b6accda8a399ec3c66df3d19217059f62903be3c74eadb9c15d5.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "INSERT INTO draft (workspace_id, path, typ, value, created_at, email)\n SELECT $2, path, typ,\n CASE WHEN typ IN ('script', 'flow')\n THEN to_json(to_jsonb(value) - 'on_behalf_of')\n ELSE value END,\n created_at, email\n FROM draft\n WHERE workspace_id = $1 AND (email = $3 OR email IS NULL)", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Text", - "Varchar", - "Text" - ] - }, - "nullable": [] - }, - "hash": "87afd5289e54b6accda8a399ec3c66df3d19217059f62903be3c74eadb9c15d5" -} diff --git a/backend/.sqlx/query-8ce4461e7dd087782a239972c173786891eb99e9ba86b11cfb17d28856babd2e.json b/backend/.sqlx/query-8ce4461e7dd087782a239972c173786891eb99e9ba86b11cfb17d28856babd2e.json new file mode 100644 index 0000000000..c732c4549d --- /dev/null +++ b/backend/.sqlx/query-8ce4461e7dd087782a239972c173786891eb99e9ba86b11cfb17d28856babd2e.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE remote_deploy_token SET token = $1 WHERE workspace_id = 'test-workspace'", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [] + }, + "hash": "8ce4461e7dd087782a239972c173786891eb99e9ba86b11cfb17d28856babd2e" +} diff --git a/backend/.sqlx/query-3c057e0284f1219865b8f7dc6eb3c1293c4b72d0b52abeefd1e954617984a2d8.json b/backend/.sqlx/query-91e2456814ac29534f5a1f65a3bb97462276930f0a83ffb458f7cb26035c8c3d.json similarity index 59% rename from backend/.sqlx/query-3c057e0284f1219865b8f7dc6eb3c1293c4b72d0b52abeefd1e954617984a2d8.json rename to backend/.sqlx/query-91e2456814ac29534f5a1f65a3bb97462276930f0a83ffb458f7cb26035c8c3d.json index 3206ba4433..57dbc41a9f 100644 --- a/backend/.sqlx/query-3c057e0284f1219865b8f7dc6eb3c1293c4b72d0b52abeefd1e954617984a2d8.json +++ b/backend/.sqlx/query-91e2456814ac29534f5a1f65a3bb97462276930f0a83ffb458f7cb26035c8c3d.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "DELETE FROM datatable_migrations WHERE workspace_id = $1 AND datatable = $2 AND timestamp = $3 RETURNING name", + "query": "DELETE FROM datatable_migrations WHERE workspace_id = $1 AND datatable = $2 AND timestamp = $3 AND code_up = $4 AND code_down IS NOT DISTINCT FROM $5 RETURNING name", "describe": { "columns": [ { @@ -13,12 +13,14 @@ "Left": [ "Text", "Text", - "Int8" + "Int8", + "Text", + "Text" ] }, "nullable": [ false ] }, - "hash": "3c057e0284f1219865b8f7dc6eb3c1293c4b72d0b52abeefd1e954617984a2d8" + "hash": "91e2456814ac29534f5a1f65a3bb97462276930f0a83ffb458f7cb26035c8c3d" } diff --git a/backend/.sqlx/query-960d6eca64de008f821b1e96345530f057d91c0e0986fdc0de7d9a3ef0c143e2.json b/backend/.sqlx/query-960d6eca64de008f821b1e96345530f057d91c0e0986fdc0de7d9a3ef0c143e2.json new file mode 100644 index 0000000000..7d819b9b84 --- /dev/null +++ b/backend/.sqlx/query-960d6eca64de008f821b1e96345530f057d91c0e0986fdc0de7d9a3ef0c143e2.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE remote_deploy_token SET workspace_id = $1 WHERE workspace_id = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Text" + ] + }, + "nullable": [] + }, + "hash": "960d6eca64de008f821b1e96345530f057d91c0e0986fdc0de7d9a3ef0c143e2" +} diff --git a/backend/.sqlx/query-965f9fb9d73f21695cf9c4018107bf8dec0a9b1c38a6776f8b3db7f2f8a883b7.json b/backend/.sqlx/query-965f9fb9d73f21695cf9c4018107bf8dec0a9b1c38a6776f8b3db7f2f8a883b7.json new file mode 100644 index 0000000000..a7fe120abb --- /dev/null +++ b/backend/.sqlx/query-965f9fb9d73f21695cf9c4018107bf8dec0a9b1c38a6776f8b3db7f2f8a883b7.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE remote_deploy_token SET connected_at = clock_timestamp() + interval '1 hour'\n WHERE workspace_id = 'test-workspace' AND email = 'test@windmill.dev'", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "965f9fb9d73f21695cf9c4018107bf8dec0a9b1c38a6776f8b3db7f2f8a883b7" +} diff --git a/backend/.sqlx/query-9723edfd6de38cf0bf21123a46d88fa7d0b650b5cfd80c099f785f5e411062e5.json b/backend/.sqlx/query-9723edfd6de38cf0bf21123a46d88fa7d0b650b5cfd80c099f785f5e411062e5.json new file mode 100644 index 0000000000..4704ddd84e --- /dev/null +++ b/backend/.sqlx/query-9723edfd6de38cf0bf21123a46d88fa7d0b650b5cfd80c099f785f5e411062e5.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM draft_move WHERE workspace_id = $1 AND typ::text = ANY($2::text[])\n AND (old_path = $3 OR (new_path = $3 AND $4::text IS NULL))", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "TextArray", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "9723edfd6de38cf0bf21123a46d88fa7d0b650b5cfd80c099f785f5e411062e5" +} diff --git a/backend/.sqlx/query-9a2345e35c9f44578aa7dde994fdde0be9e0d192e0aee224bff78d8bd2ccbd8b.json b/backend/.sqlx/query-9a2345e35c9f44578aa7dde994fdde0be9e0d192e0aee224bff78d8bd2ccbd8b.json new file mode 100644 index 0000000000..0bc09d9430 --- /dev/null +++ b/backend/.sqlx/query-9a2345e35c9f44578aa7dde994fdde0be9e0d192e0aee224bff78d8bd2ccbd8b.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE v2_job_completed SET status = 'canceled'::job_status, canceled_by = $2, canceled_reason = $3 WHERE id = $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Uuid", + "Varchar", + "Text" + ] + }, + "nullable": [] + }, + "hash": "9a2345e35c9f44578aa7dde994fdde0be9e0d192e0aee224bff78d8bd2ccbd8b" +} diff --git a/backend/.sqlx/query-9b61979cc78e8b7b63ffa6a59cf3d29ae8b924200c46da457a9346379c60fa9f.json b/backend/.sqlx/query-9b61979cc78e8b7b63ffa6a59cf3d29ae8b924200c46da457a9346379c60fa9f.json new file mode 100644 index 0000000000..112ca1603f --- /dev/null +++ b/backend/.sqlx/query-9b61979cc78e8b7b63ffa6a59cf3d29ae8b924200c46da457a9346379c60fa9f.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE workspace_settings SET remote_deploy_target = $1::jsonb,\n remote_deploy_target_changed_at = CASE\n WHEN remote_deploy_target IS DISTINCT FROM $1::jsonb THEN clock_timestamp()\n ELSE remote_deploy_target_changed_at END\n WHERE workspace_id = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Jsonb", + "Text" + ] + }, + "nullable": [] + }, + "hash": "9b61979cc78e8b7b63ffa6a59cf3d29ae8b924200c46da457a9346379c60fa9f" +} diff --git a/backend/.sqlx/query-9d1b202844e6935e636d1570a92d46af4419dda9410ffc46f19bc38e56efe609.json b/backend/.sqlx/query-9d1b202844e6935e636d1570a92d46af4419dda9410ffc46f19bc38e56efe609.json new file mode 100644 index 0000000000..f882ce2986 --- /dev/null +++ b/backend/.sqlx/query-9d1b202844e6935e636d1570a92d46af4419dda9410ffc46f19bc38e56efe609.json @@ -0,0 +1,18 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM draft_move\n WHERE workspace_id = $1 AND typ::text = ANY($2::text[])\n AND new_path = $4 AND old_path <> $3 AND ($5::text IS NULL OR email = $5)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "TextArray", + "Text", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "9d1b202844e6935e636d1570a92d46af4419dda9410ffc46f19bc38e56efe609" +} diff --git a/backend/.sqlx/query-a0f1c0df6bc2f1fbca50edee90e42c94445536e201b322eda6f7a90bdf38f36a.json b/backend/.sqlx/query-a0f1c0df6bc2f1fbca50edee90e42c94445536e201b322eda6f7a90bdf38f36a.json deleted file mode 100644 index 3ed4f5a316..0000000000 --- a/backend/.sqlx/query-a0f1c0df6bc2f1fbca50edee90e42c94445536e201b322eda6f7a90bdf38f36a.json +++ /dev/null @@ -1,35 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT flow_version.id, flow_version.created_at, deployment_metadata.deployment_msg FROM flow_version \n LEFT JOIN deployment_metadata ON flow_version.id = deployment_metadata.flow_version\n WHERE flow_version.path = $1 AND flow_version.workspace_id = $2 \n ORDER BY flow_version.created_at DESC", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "id", - "type_info": "Int8" - }, - { - "ordinal": 1, - "name": "created_at", - "type_info": "Timestamptz" - }, - { - "ordinal": 2, - "name": "deployment_msg", - "type_info": "Text" - } - ], - "parameters": { - "Left": [ - "Text", - "Text" - ] - }, - "nullable": [ - false, - false, - true - ] - }, - "hash": "a0f1c0df6bc2f1fbca50edee90e42c94445536e201b322eda6f7a90bdf38f36a" -} diff --git a/backend/.sqlx/query-a1ed94ef0a6d4f7bc89ac4a2341bc06dc95d60ec92469523935f5a18a4d3f907.json b/backend/.sqlx/query-a1ed94ef0a6d4f7bc89ac4a2341bc06dc95d60ec92469523935f5a18a4d3f907.json new file mode 100644 index 0000000000..c536425f1d --- /dev/null +++ b/backend/.sqlx/query-a1ed94ef0a6d4f7bc89ac4a2341bc06dc95d60ec92469523935f5a18a4d3f907.json @@ -0,0 +1,85 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT\n -- The guard's own predicate, ordered own row, then legacy, then another\n -- user's other-kind row: each needs a different remedy, and a bare\n -- LIMIT 1 would name an arbitrary one.\n (SELECT typ::text FROM draft WHERE workspace_id = $1 AND path = $3\n AND typ::text = ANY($6::text[])\n AND (typ <> $2 OR email = $4 OR email IS NULL)\n ORDER BY CASE WHEN email = $4 THEN 0 WHEN email IS NULL THEN 1 ELSE 2 END\n LIMIT 1) as \"at_target\",\n EXISTS(SELECT 1 FROM draft WHERE workspace_id = $1 AND path = $3\n AND typ::text = ANY($6::text[]) AND email = $4) as \"at_target_own!\",\n EXISTS(SELECT 1 FROM draft WHERE workspace_id = $1 AND path = $3\n AND typ::text = ANY($6::text[]) AND email IS NULL) as \"at_target_legacy!\",\n EXISTS(SELECT 1 FROM draft WHERE workspace_id = $1 AND path = $5\n AND typ = $2 AND email = $4\n AND position(chr(92) || 'u0000' in replace(value::text, chr(92) || chr(92), '')) > 0\n ) as \"poisoned!\",\n EXISTS(SELECT 1 FROM draft WHERE workspace_id = $1 AND path = $5\n AND typ = $2 AND email IS NULL) as \"legacy!\" ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "at_target", + "type_info": "Text" + }, + { + "ordinal": 1, + "name": "at_target_own!", + "type_info": "Bool" + }, + { + "ordinal": 2, + "name": "at_target_legacy!", + "type_info": "Bool" + }, + { + "ordinal": 3, + "name": "poisoned!", + "type_info": "Bool" + }, + { + "ordinal": 4, + "name": "legacy!", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text", + { + "Custom": { + "name": "draft_kind", + "kind": { + "Enum": [ + "script", + "flow", + "app", + "raw_app", + "resource", + "variable", + "trigger_schedule", + "trigger_webhook", + "trigger_default_email", + "trigger_email", + "trigger_http", + "trigger_websocket", + "trigger_postgres", + "trigger_kafka", + "trigger_nats", + "trigger_mqtt", + "trigger_sqs", + "trigger_gcp", + "trigger_azure", + "trigger_poll", + "trigger_cli", + "trigger_nextcloud", + "trigger_google", + "trigger_github", + "data_pipeline", + "trigger_amqp" + ] + } + } + }, + "Text", + "Text", + "Text", + "TextArray" + ] + }, + "nullable": [ + null, + null, + null, + null, + null + ] + }, + "hash": "a1ed94ef0a6d4f7bc89ac4a2341bc06dc95d60ec92469523935f5a18a4d3f907" +} diff --git a/backend/.sqlx/query-a725d5b99b65a1637406179b058a1ff36620099c6b87fd82a59eae1e35303c07.json b/backend/.sqlx/query-a725d5b99b65a1637406179b058a1ff36620099c6b87fd82a59eae1e35303c07.json new file mode 100644 index 0000000000..2acc85fcd4 --- /dev/null +++ b/backend/.sqlx/query-a725d5b99b65a1637406179b058a1ff36620099c6b87fd82a59eae1e35303c07.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE draft_move SET workspace_id = $1 WHERE workspace_id = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Text" + ] + }, + "nullable": [] + }, + "hash": "a725d5b99b65a1637406179b058a1ff36620099c6b87fd82a59eae1e35303c07" +} diff --git a/backend/.sqlx/query-a866da25370c8b829fd8038c55c0f43e27a6eea453415495b66c50045defacff.json b/backend/.sqlx/query-a866da25370c8b829fd8038c55c0f43e27a6eea453415495b66c50045defacff.json new file mode 100644 index 0000000000..18951e2a87 --- /dev/null +++ b/backend/.sqlx/query-a866da25370c8b829fd8038c55c0f43e27a6eea453415495b66c50045defacff.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM usr WHERE workspace_id = 'test-workspace' AND email = 'test@windmill.dev'", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "a866da25370c8b829fd8038c55c0f43e27a6eea453415495b66c50045defacff" +} diff --git a/backend/.sqlx/query-a9ae6c222727b7078fd25ed91f51c363a4e5719b5ceb17e8194c2eb038f59c68.json b/backend/.sqlx/query-a9ae6c222727b7078fd25ed91f51c363a4e5719b5ceb17e8194c2eb038f59c68.json new file mode 100644 index 0000000000..6529400823 --- /dev/null +++ b/backend/.sqlx/query-a9ae6c222727b7078fd25ed91f51c363a4e5719b5ceb17e8194c2eb038f59c68.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE remote_deploy_token SET token = 'not-under-this-key'\n WHERE workspace_id = 'test-workspace'", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "a9ae6c222727b7078fd25ed91f51c363a4e5719b5ceb17e8194c2eb038f59c68" +} diff --git a/backend/.sqlx/query-af1bc312f13d7590a138e2de4996dab8e5ac9c774cf81df9da85ac3c40e8b3c7.json b/backend/.sqlx/query-af1bc312f13d7590a138e2de4996dab8e5ac9c774cf81df9da85ac3c40e8b3c7.json new file mode 100644 index 0000000000..4ec7511c7c --- /dev/null +++ b/backend/.sqlx/query-af1bc312f13d7590a138e2de4996dab8e5ac9c774cf81df9da85ac3c40e8b3c7.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO token (token_hash, token_prefix, token, email, label, super_admin)\n VALUES (encode(sha256('GONE_TOKEN'::bytea), 'hex'), 'GONE_TOKEN', 'GONE_TOKEN',\n 'test@windmill.dev', 'gone', true)", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "af1bc312f13d7590a138e2de4996dab8e5ac9c774cf81df9da85ac3c40e8b3c7" +} diff --git a/backend/.sqlx/query-b2014ff45992b34c08f27419c26e975ff2b1baee63651e0ce79156b331389923.json b/backend/.sqlx/query-b2014ff45992b34c08f27419c26e975ff2b1baee63651e0ce79156b331389923.json new file mode 100644 index 0000000000..77cd542d6b --- /dev/null +++ b/backend/.sqlx/query-b2014ff45992b34c08f27419c26e975ff2b1baee63651e0ce79156b331389923.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM remote_deploy_token WHERE email = $1 AND workspace_id = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "b2014ff45992b34c08f27419c26e975ff2b1baee63651e0ce79156b331389923" +} diff --git a/backend/.sqlx/query-383c80239525d9c4ee90e2f5cb6e46c8a0983c52ad15db083f84a9e344dc4c99.json b/backend/.sqlx/query-b30fb28004897cd1d201a416cdabf124a055eb44b3c5892fd18ff7baf6010ae9.json similarity index 79% rename from backend/.sqlx/query-383c80239525d9c4ee90e2f5cb6e46c8a0983c52ad15db083f84a9e344dc4c99.json rename to backend/.sqlx/query-b30fb28004897cd1d201a416cdabf124a055eb44b3c5892fd18ff7baf6010ae9.json index 8a780ccda7..c15e0edb33 100644 --- a/backend/.sqlx/query-383c80239525d9c4ee90e2f5cb6e46c8a0983c52ad15db083f84a9e344dc4c99.json +++ b/backend/.sqlx/query-b30fb28004897cd1d201a416cdabf124a055eb44b3c5892fd18ff7baf6010ae9.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "UPDATE token SET label = $1\n WHERE email = $2 AND token_prefix = $3\n AND (label IS NULL OR (\n label <> 'session'\n AND label <> 'guest_session'\n AND lower(label) NOT LIKE 'ephemeral%'\n AND label <> 'debugger-token'\n AND label NOT LIKE 'mcp-oauth-%'\n AND NOT starts_with(label, 'embed_app:')\n AND NOT starts_with(label, 'sdk_app:')\n AND NOT starts_with(label, 'impersonation:')\n AND NOT starts_with(label, 'cli-login:')\n ))\n RETURNING token_prefix", + "query": "UPDATE token SET label = $1\n WHERE email = $2 AND token_prefix = $3\n AND (label IS NULL OR (\n label <> 'session'\n AND label <> 'guest_session'\n AND lower(label) NOT LIKE 'ephemeral%'\n AND label <> 'debugger-token'\n AND label NOT LIKE 'mcp-oauth-%'\n AND NOT starts_with(label, 'embed_app:')\n AND NOT starts_with(label, 'sdk_app:')\n AND NOT starts_with(label, 'impersonation:')\n AND NOT starts_with(label, 'cli-login:')\n AND NOT starts_with(label, 'remote-deploy:')\n ))\n RETURNING token_prefix", "describe": { "columns": [ { @@ -20,5 +20,5 @@ false ] }, - "hash": "383c80239525d9c4ee90e2f5cb6e46c8a0983c52ad15db083f84a9e344dc4c99" + "hash": "b30fb28004897cd1d201a416cdabf124a055eb44b3c5892fd18ff7baf6010ae9" } diff --git a/backend/.sqlx/query-bd8ac169ad15346f28f561233fe340efd453512c1464a413af2393ab8bfea9ee.json b/backend/.sqlx/query-bd8ac169ad15346f28f561233fe340efd453512c1464a413af2393ab8bfea9ee.json new file mode 100644 index 0000000000..8a59831e8a --- /dev/null +++ b/backend/.sqlx/query-bd8ac169ad15346f28f561233fe340efd453512c1464a413af2393ab8bfea9ee.json @@ -0,0 +1,47 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT a.id as app_id, av.id as version_id, dm.deployment_msg as deployment_msg,\n av.created_by as created_by, av.created_at as created_at\n FROM app a JOIN app_version av\n ON av.id = a.versions[array_upper(a.versions, 1)] AND av.app_id = a.id\n LEFT JOIN deployment_metadata dm ON av.id = dm.app_version\n WHERE a.workspace_id = $1 AND a.path = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "app_id", + "type_info": "Int8" + }, + { + "ordinal": 1, + "name": "version_id", + "type_info": "Int8" + }, + { + "ordinal": 2, + "name": "deployment_msg", + "type_info": "Text" + }, + { + "ordinal": 3, + "name": "created_by", + "type_info": "Varchar" + }, + { + "ordinal": 4, + "name": "created_at", + "type_info": "Timestamptz" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false, + false, + true, + false, + false + ] + }, + "hash": "bd8ac169ad15346f28f561233fe340efd453512c1464a413af2393ab8bfea9ee" +} diff --git a/backend/.sqlx/query-c92cc71e6d10c41368f7aa75b0799c2e1e9ca0ed33077ade8d7560e7cc21fa06.json b/backend/.sqlx/query-c92cc71e6d10c41368f7aa75b0799c2e1e9ca0ed33077ade8d7560e7cc21fa06.json deleted file mode 100644 index 8a4b957c2a..0000000000 --- a/backend/.sqlx/query-c92cc71e6d10c41368f7aa75b0799c2e1e9ca0ed33077ade8d7560e7cc21fa06.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "DELETE FROM v2_job_queue WHERE id = $1 RETURNING 1", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "?column?", - "type_info": "Int4" - } - ], - "parameters": { - "Left": [ - "Uuid" - ] - }, - "nullable": [ - null - ] - }, - "hash": "c92cc71e6d10c41368f7aa75b0799c2e1e9ca0ed33077ade8d7560e7cc21fa06" -} diff --git a/backend/.sqlx/query-c96486a83ba0f4bf8daf3c525d6f67554643d0e730bec2e64840ad3d1862e540.json b/backend/.sqlx/query-c96486a83ba0f4bf8daf3c525d6f67554643d0e730bec2e64840ad3d1862e540.json new file mode 100644 index 0000000000..7422dd2333 --- /dev/null +++ b/backend/.sqlx/query-c96486a83ba0f4bf8daf3c525d6f67554643d0e730bec2e64840ad3d1862e540.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM remote_deploy_token WHERE email = 'test@windmill.dev'", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "c96486a83ba0f4bf8daf3c525d6f67554643d0e730bec2e64840ad3d1862e540" +} diff --git a/backend/.sqlx/query-cad45b99cd110589665e864f73403c2b12a3cf12ee81e67b0e9e520b3315e7d6.json b/backend/.sqlx/query-cad45b99cd110589665e864f73403c2b12a3cf12ee81e67b0e9e520b3315e7d6.json new file mode 100644 index 0000000000..85c999e3ba --- /dev/null +++ b/backend/.sqlx/query-cad45b99cd110589665e864f73403c2b12a3cf12ee81e67b0e9e520b3315e7d6.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE workspace_settings\n SET remote_deploy_target = $1, remote_deploy_target_changed_at = now()\n WHERE workspace_id = 'test-workspace'", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Jsonb" + ] + }, + "nullable": [] + }, + "hash": "cad45b99cd110589665e864f73403c2b12a3cf12ee81e67b0e9e520b3315e7d6" +} diff --git a/backend/.sqlx/query-d0548225d92e6e7d0eb9a0227a6522398d2a3ae99d0568b785e63749d6f3225a.json b/backend/.sqlx/query-d0548225d92e6e7d0eb9a0227a6522398d2a3ae99d0568b785e63749d6f3225a.json new file mode 100644 index 0000000000..2ff2722f77 --- /dev/null +++ b/backend/.sqlx/query-d0548225d92e6e7d0eb9a0227a6522398d2a3ae99d0568b785e63749d6f3225a.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE workspace_settings ws\n SET datatable = (\n SELECT jsonb_set(ws.datatable, '{datatables}', jsonb_object_agg(\n dt.key,\n (SELECT CASE WHEN r.v->'governed_by'->>'workspace_id' = $2\n THEN jsonb_set(r.v, '{governed_by,workspace_id}', to_jsonb($1::text))\n ELSE r.v END\n FROM (SELECT CASE WHEN dt.value->'reference'->>'workspace_id' = $2\n THEN jsonb_set(dt.value, '{reference,workspace_id}', to_jsonb($1::text))\n ELSE dt.value END AS v) r)\n ))\n FROM jsonb_each(ws.datatable->'datatables') dt\n )\n WHERE jsonb_typeof(ws.datatable->'datatables') = 'object'\n AND (ws.datatable::text LIKE '%\"reference\"%'\n OR ws.datatable::text LIKE '%\"governed_by\"%')", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "d0548225d92e6e7d0eb9a0227a6522398d2a3ae99d0568b785e63749d6f3225a" +} diff --git a/backend/.sqlx/query-d21d9d7e37045529ae4b73febc145cb080009a7a6d9ef33111755b470ff5abe2.json b/backend/.sqlx/query-d21d9d7e37045529ae4b73febc145cb080009a7a6d9ef33111755b470ff5abe2.json new file mode 100644 index 0000000000..cde3e698d9 --- /dev/null +++ b/backend/.sqlx/query-d21d9d7e37045529ae4b73febc145cb080009a7a6d9ef33111755b470ff5abe2.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM remote_deploy_token WHERE workspace_id = $1 AND email = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "d21d9d7e37045529ae4b73febc145cb080009a7a6d9ef33111755b470ff5abe2" +} diff --git a/backend/.sqlx/query-d305aed6172e25707492677b6cd3b8f3f6ea8f2c8f5bdc1a97736c8e52378ee9.json b/backend/.sqlx/query-d305aed6172e25707492677b6cd3b8f3f6ea8f2c8f5bdc1a97736c8e52378ee9.json new file mode 100644 index 0000000000..4031e19231 --- /dev/null +++ b/backend/.sqlx/query-d305aed6172e25707492677b6cd3b8f3f6ea8f2c8f5bdc1a97736c8e52378ee9.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE usr SET created_at = created_at - interval '1 hour'\n WHERE workspace_id = 'test-workspace' AND email = 'test2@windmill.dev'", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "d305aed6172e25707492677b6cd3b8f3f6ea8f2c8f5bdc1a97736c8e52378ee9" +} diff --git a/backend/.sqlx/query-8aae160c589adf02e20b7e6ba860b66fb08f156776b96b7f3701aed330a7d000.json b/backend/.sqlx/query-d50d7a93f5c97570e3ef51ef542de41e9c1a9230911a2331d7127572722e340e.json similarity index 54% rename from backend/.sqlx/query-8aae160c589adf02e20b7e6ba860b66fb08f156776b96b7f3701aed330a7d000.json rename to backend/.sqlx/query-d50d7a93f5c97570e3ef51ef542de41e9c1a9230911a2331d7127572722e340e.json index d05c5b11af..edc0efd9ca 100644 --- a/backend/.sqlx/query-8aae160c589adf02e20b7e6ba860b66fb08f156776b96b7f3701aed330a7d000.json +++ b/backend/.sqlx/query-d50d7a93f5c97570e3ef51ef542de41e9c1a9230911a2331d7127572722e340e.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "WITH active_users AS (SELECT distinct username as email FROM (SELECT username, timestamp, operation FROM audit_partitioned UNION ALL SELECT username, timestamp, operation FROM audit) AS a WHERE timestamp > NOW() - INTERVAL '1 month' AND (operation = 'users.login' OR operation = 'oauth.login' OR operation = 'users.token.refresh')),\n authors as (SELECT distinct email FROM usr WHERE usr.operator IS false)\n SELECT email as \"email!\", (email NOT IN (SELECT email FROM authors)) as operator_only, NULL::bool as is_workspace_admin, login_type::text, verified as \"verified!\", super_admin as \"super_admin!\", devops as \"devops!\", name, company, username, first_time_user as \"first_time_user!\", role_source as \"role_source!\", disabled as \"disabled!\", NULL::text as workspace_id\n FROM password\n WHERE email IN (SELECT email FROM active_users)\n UNION ALL\n SELECT email as \"email!\", operator as operator_only, is_admin as is_workspace_admin, 'service_account'::text as login_type, true as \"verified!\", false as \"super_admin!\", false as \"devops!\", NULL::text as name, NULL::text as company, username, false as \"first_time_user!\", 'service_account'::text as \"role_source!\", disabled as \"disabled!\", workspace_id\n FROM usr\n WHERE is_service_account IS true\n ORDER BY \"super_admin!\" DESC, \"devops!\" DESC\n LIMIT $1 OFFSET $2", + "query": "WITH active_users AS (SELECT distinct username as email FROM audit_partitioned WHERE timestamp > NOW() - INTERVAL '1 month' AND (operation = 'users.login' OR operation = 'oauth.login' OR operation = 'users.token.refresh')),\n authors as (SELECT distinct email FROM usr WHERE usr.operator IS false)\n SELECT email as \"email!\", (email NOT IN (SELECT email FROM authors)) as operator_only, NULL::bool as is_workspace_admin, login_type::text, verified as \"verified!\", super_admin as \"super_admin!\", devops as \"devops!\", name, company, username, first_time_user as \"first_time_user!\", role_source as \"role_source!\", disabled as \"disabled!\", NULL::text as workspace_id\n FROM password\n WHERE email IN (SELECT email FROM active_users)\n UNION ALL\n SELECT email as \"email!\", operator as operator_only, is_admin as is_workspace_admin, 'service_account'::text as login_type, true as \"verified!\", false as \"super_admin!\", false as \"devops!\", NULL::text as name, NULL::text as company, username, false as \"first_time_user!\", 'service_account'::text as \"role_source!\", disabled as \"disabled!\", workspace_id\n FROM usr\n WHERE is_service_account IS true\n ORDER BY \"super_admin!\" DESC, \"devops!\" DESC\n LIMIT $1 OFFSET $2", "describe": { "columns": [ { @@ -97,5 +97,5 @@ null ] }, - "hash": "8aae160c589adf02e20b7e6ba860b66fb08f156776b96b7f3701aed330a7d000" + "hash": "d50d7a93f5c97570e3ef51ef542de41e9c1a9230911a2331d7127572722e340e" } diff --git a/backend/.sqlx/query-d8186f0cee285aa50db7626409aec7e0504b068ffa8bb185d9384ce1422fd3d1.json b/backend/.sqlx/query-d8186f0cee285aa50db7626409aec7e0504b068ffa8bb185d9384ce1422fd3d1.json deleted file mode 100644 index 70872a1328..0000000000 --- a/backend/.sqlx/query-d8186f0cee285aa50db7626409aec7e0504b068ffa8bb185d9384ce1422fd3d1.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "DELETE FROM audit WHERE timestamp <= now() - ($1::bigint::text || ' s')::interval", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Int8" - ] - }, - "nullable": [] - }, - "hash": "d8186f0cee285aa50db7626409aec7e0504b068ffa8bb185d9384ce1422fd3d1" -} diff --git a/backend/.sqlx/query-dc37f9ad685a2f2bd781be8678e0548a860815f30e866362344434f23aa7c83c.json b/backend/.sqlx/query-dc37f9ad685a2f2bd781be8678e0548a860815f30e866362344434f23aa7c83c.json new file mode 100644 index 0000000000..02d889c38b --- /dev/null +++ b/backend/.sqlx/query-dc37f9ad685a2f2bd781be8678e0548a860815f30e866362344434f23aa7c83c.json @@ -0,0 +1,38 @@ +{ + "db_name": "PostgreSQL", + "query": "WITH active_users as (SELECT distinct username as email FROM audit_partitioned WHERE timestamp > NOW() - INTERVAL '1 month' AND (operation = 'users.login' OR operation = 'oauth.login' OR operation = 'users.token.refresh')),\n active_authors as (SELECT distinct email FROM usr WHERE usr.operator IS false AND email IN (SELECT email FROM active_users)),\n active_authors_agg as (SELECT array_agg(email) as authors FROM active_authors),\n active_ops_agg as (SELECT array_agg(email) as operators from active_users WHERE email NOT IN (SELECT email FROM active_authors))\n SELECT active_authors_agg.authors, active_ops_agg.operators, array_length(active_authors_agg.authors, 1) as author_count, array_length(active_ops_agg.operators, 1) as operator_count FROM active_authors_agg, active_ops_agg", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "authors", + "type_info": "VarcharArray" + }, + { + "ordinal": 1, + "name": "operators", + "type_info": "VarcharArray" + }, + { + "ordinal": 2, + "name": "author_count", + "type_info": "Int4" + }, + { + "ordinal": 3, + "name": "operator_count", + "type_info": "Int4" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + null, + null, + null, + null + ] + }, + "hash": "dc37f9ad685a2f2bd781be8678e0548a860815f30e866362344434f23aa7c83c" +} diff --git a/backend/.sqlx/query-dfa54341a687431ea46ec842ab4167bcfd875cb4646ef859a7b855428bf48290.json b/backend/.sqlx/query-dfa54341a687431ea46ec842ab4167bcfd875cb4646ef859a7b855428bf48290.json new file mode 100644 index 0000000000..7acdcf8801 --- /dev/null +++ b/backend/.sqlx/query-dfa54341a687431ea46ec842ab4167bcfd875cb4646ef859a7b855428bf48290.json @@ -0,0 +1,38 @@ +{ + "db_name": "PostgreSQL", + "query": "\n WITH capped AS (\n SELECT timestamp, operation, resource, parameters\n FROM audit_partitioned\n WHERE workspace_id = 'admins'\n AND operation = 'workspace_fairness.capped'\n ORDER BY timestamp DESC\n LIMIT 200\n ), uncapped AS (\n SELECT timestamp, operation, resource, parameters\n FROM audit_partitioned\n WHERE workspace_id = 'admins'\n AND operation = 'workspace_fairness.uncapped'\n ORDER BY timestamp DESC\n LIMIT 200\n )\n SELECT timestamp AS \"timestamp!\",\n operation::text AS \"operation!\",\n resource AS workspace_id,\n parameters\n FROM (SELECT * FROM capped UNION ALL SELECT * FROM uncapped) e\n ORDER BY timestamp DESC\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "timestamp!", + "type_info": "Timestamptz" + }, + { + "ordinal": 1, + "name": "operation!", + "type_info": "Text" + }, + { + "ordinal": 2, + "name": "workspace_id", + "type_info": "Varchar" + }, + { + "ordinal": 3, + "name": "parameters", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + null, + null, + null, + null + ] + }, + "hash": "dfa54341a687431ea46ec842ab4167bcfd875cb4646ef859a7b855428bf48290" +} diff --git a/backend/.sqlx/query-e2760debbb69182f1385bfe8e18933eeff7396430607c02cbd0cc682d5c34d61.json b/backend/.sqlx/query-e2760debbb69182f1385bfe8e18933eeff7396430607c02cbd0cc682d5c34d61.json new file mode 100644 index 0000000000..9930e0494f --- /dev/null +++ b/backend/.sqlx/query-e2760debbb69182f1385bfe8e18933eeff7396430607c02cbd0cc682d5c34d61.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT canceled_by IS NOT NULL AS \"canceled!\" FROM v2_job_completed WHERE id = $1 AND workspace_id = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "canceled!", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "e2760debbb69182f1385bfe8e18933eeff7396430607c02cbd0cc682d5c34d61" +} diff --git a/backend/.sqlx/query-e428f0cdb9077bffe2b7271d9b5f30ed81e8c26a9f1e65f423078947153ff9dd.json b/backend/.sqlx/query-e428f0cdb9077bffe2b7271d9b5f30ed81e8c26a9f1e65f423078947153ff9dd.json new file mode 100644 index 0000000000..57b5e2abd1 --- /dev/null +++ b/backend/.sqlx/query-e428f0cdb9077bffe2b7271d9b5f30ed81e8c26a9f1e65f423078947153ff9dd.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE draft_move SET email = $1 WHERE email = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Text" + ] + }, + "nullable": [] + }, + "hash": "e428f0cdb9077bffe2b7271d9b5f30ed81e8c26a9f1e65f423078947153ff9dd" +} diff --git a/backend/.sqlx/query-e524e89440004953cc0b1cb57b8df6bc09f19cf53b141fac5aefa981463d39fa.json b/backend/.sqlx/query-e524e89440004953cc0b1cb57b8df6bc09f19cf53b141fac5aefa981463d39fa.json new file mode 100644 index 0000000000..af8d0eb1ab --- /dev/null +++ b/backend/.sqlx/query-e524e89440004953cc0b1cb57b8df6bc09f19cf53b141fac5aefa981463d39fa.json @@ -0,0 +1,29 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT ws.workspace_id AS \"workspace_id!\", dt.key AS \"datatable!\"\n FROM workspace_settings ws\n CROSS JOIN LATERAL jsonb_each(COALESCE(ws.datatable->'datatables', '{}'::jsonb)) dt\n WHERE EXISTS (\n SELECT 1 FROM (VALUES ('reference'), ('governed_by')) k(link)\n WHERE dt.value->k.link->>'workspace_id' = $1\n AND dt.value->k.link->>'datatable' = $2\n )", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "workspace_id!", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "datatable!", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false, + null + ] + }, + "hash": "e524e89440004953cc0b1cb57b8df6bc09f19cf53b141fac5aefa981463d39fa" +} diff --git a/backend/.sqlx/query-e6140682ebc65083f48714ab3299fdb239ef507e23003db03aff7d35b02c540b.json b/backend/.sqlx/query-e6140682ebc65083f48714ab3299fdb239ef507e23003db03aff7d35b02c540b.json new file mode 100644 index 0000000000..f218653d46 --- /dev/null +++ b/backend/.sqlx/query-e6140682ebc65083f48714ab3299fdb239ef507e23003db03aff7d35b02c540b.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO draft (workspace_id, path, typ, value, created_at, email, base)\n SELECT $2, path, typ,\n to_json(\n CASE WHEN typ IN ('script', 'flow') THEN clean - 'on_behalf_of' ELSE clean END\n - CASE WHEN typ = 'flow' THEN 'version_id'\n WHEN typ IN ('app', 'raw_app') THEN 'parent_version'\n ELSE '' END\n ),\n created_at, email,\n CASE WHEN typ = 'script' THEN base END\n FROM (\n SELECT d.path, d.typ, d.created_at, d.email, d.base,\n replace(replace(replace(d.value::text, chr(92) || chr(92), chr(1)), chr(92) || 'u0000', ''), chr(1), chr(92) || chr(92))::jsonb AS clean\n FROM draft d\n WHERE d.workspace_id = $1 AND (d.email = $3 OR d.email IS NULL)\n ) s", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Varchar", + "Text" + ] + }, + "nullable": [] + }, + "hash": "e6140682ebc65083f48714ab3299fdb239ef507e23003db03aff7d35b02c540b" +} diff --git a/backend/.sqlx/query-e63da92247045f2aaaa62b9d6ee17510fd05d0b7c5311b61e7257ccd07b16da2.json b/backend/.sqlx/query-e63da92247045f2aaaa62b9d6ee17510fd05d0b7c5311b61e7257ccd07b16da2.json new file mode 100644 index 0000000000..926add8159 --- /dev/null +++ b/backend/.sqlx/query-e63da92247045f2aaaa62b9d6ee17510fd05d0b7c5311b61e7257ccd07b16da2.json @@ -0,0 +1,59 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT m.new_path FROM draft_move m\n WHERE m.workspace_id = $1 AND m.typ = $2 AND m.old_path = $3\n AND (m.email IS NULL OR m.email = $4)\n AND NOT EXISTS (\n SELECT 1 FROM draft d\n WHERE d.workspace_id = $1 AND d.typ = $2 AND d.path = $3\n AND d.email IS NOT DISTINCT FROM $4\n )\n ORDER BY m.email IS NULL\n LIMIT 1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "new_path", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text", + { + "Custom": { + "name": "draft_kind", + "kind": { + "Enum": [ + "script", + "flow", + "app", + "raw_app", + "resource", + "variable", + "trigger_schedule", + "trigger_webhook", + "trigger_default_email", + "trigger_email", + "trigger_http", + "trigger_websocket", + "trigger_postgres", + "trigger_kafka", + "trigger_nats", + "trigger_mqtt", + "trigger_sqs", + "trigger_gcp", + "trigger_azure", + "trigger_poll", + "trigger_cli", + "trigger_nextcloud", + "trigger_google", + "trigger_github", + "data_pipeline", + "trigger_amqp" + ] + } + } + }, + "Text", + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "e63da92247045f2aaaa62b9d6ee17510fd05d0b7c5311b61e7257ccd07b16da2" +} diff --git a/backend/.sqlx/query-ed2eb32d445164e7ab5fb735e01f86505bc499320e08d1aad15a9a82c3823c0b.json b/backend/.sqlx/query-ed2eb32d445164e7ab5fb735e01f86505bc499320e08d1aad15a9a82c3823c0b.json new file mode 100644 index 0000000000..b11e41fe89 --- /dev/null +++ b/backend/.sqlx/query-ed2eb32d445164e7ab5fb735e01f86505bc499320e08d1aad15a9a82c3823c0b.json @@ -0,0 +1,28 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM v2_job_queue WHERE id = $1 RETURNING canceled_by, canceled_reason", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "canceled_by", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "canceled_reason", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Uuid" + ] + }, + "nullable": [ + true, + true + ] + }, + "hash": "ed2eb32d445164e7ab5fb735e01f86505bc499320e08d1aad15a9a82c3823c0b" +} diff --git a/backend/.sqlx/query-ee3ed82783b5b7d2f41f16a1d94f006687bd9de4611203b5dd0c28eacee02842.json b/backend/.sqlx/query-ee3ed82783b5b7d2f41f16a1d94f006687bd9de4611203b5dd0c28eacee02842.json new file mode 100644 index 0000000000..a1d873778f --- /dev/null +++ b/backend/.sqlx/query-ee3ed82783b5b7d2f41f16a1d94f006687bd9de4611203b5dd0c28eacee02842.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO workspace_settings (workspace_id, slack_team_id, slack_name, slack_command_script, slack_email, customer_id, plan, webhook, ai_config, large_file_storage, git_sync, default_app, default_scripts, deploy_ui, mute_critical_alerts, color, operator_settings, teams_command_script, teams_team_id, teams_team_name, git_app_installations, git_credentials, ducklake, dbt_warehouses, slack_oauth_client_id, slack_oauth_client_secret, datatable, teams_team_guid, auto_invite, error_handler, success_handler, public_app_execution_limit_per_minute, error_handler_fallback_to_instance_alerts, guest_access_enabled, guest_jwt_public_key, guest_jwt_jwks_url, add_admins_and_developers_to_forks, remote_deploy_target, remote_deploy_target_changed_at) SELECT $1, slack_team_id, slack_name, slack_command_script, slack_email, customer_id, plan, webhook, ai_config, large_file_storage, git_sync, default_app, default_scripts, deploy_ui, mute_critical_alerts, color, operator_settings, teams_command_script, teams_team_id, teams_team_name, git_app_installations, git_credentials, ducklake, dbt_warehouses, slack_oauth_client_id, slack_oauth_client_secret, datatable, teams_team_guid, auto_invite, error_handler, success_handler, public_app_execution_limit_per_minute, error_handler_fallback_to_instance_alerts, guest_access_enabled, guest_jwt_public_key, guest_jwt_jwks_url, add_admins_and_developers_to_forks, remote_deploy_target, remote_deploy_target_changed_at FROM workspace_settings WHERE workspace_id = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Text" + ] + }, + "nullable": [] + }, + "hash": "ee3ed82783b5b7d2f41f16a1d94f006687bd9de4611203b5dd0c28eacee02842" +} diff --git a/backend/.sqlx/query-eefa0588a6a927fd9b3f65e1df652fb2b4cf7983049d2c490940df360c7e2b30.json b/backend/.sqlx/query-eefa0588a6a927fd9b3f65e1df652fb2b4cf7983049d2c490940df360c7e2b30.json deleted file mode 100644 index ec641bf32d..0000000000 --- a/backend/.sqlx/query-eefa0588a6a927fd9b3f65e1df652fb2b4cf7983049d2c490940df360c7e2b30.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "INSERT INTO workspace_settings (workspace_id, slack_team_id, slack_name, slack_command_script, slack_email, customer_id, plan, webhook, ai_config, large_file_storage, git_sync, default_app, default_scripts, deploy_ui, mute_critical_alerts, color, operator_settings, teams_command_script, teams_team_id, teams_team_name, git_app_installations, git_credentials, ducklake, dbt_warehouses, slack_oauth_client_id, slack_oauth_client_secret, datatable, teams_team_guid, auto_invite, error_handler, success_handler, public_app_execution_limit_per_minute, error_handler_fallback_to_instance_alerts, guest_access_enabled, guest_jwt_public_key, guest_jwt_jwks_url, add_admins_and_developers_to_forks) SELECT $1, slack_team_id, slack_name, slack_command_script, slack_email, customer_id, plan, webhook, ai_config, large_file_storage, git_sync, default_app, default_scripts, deploy_ui, mute_critical_alerts, color, operator_settings, teams_command_script, teams_team_id, teams_team_name, git_app_installations, git_credentials, ducklake, dbt_warehouses, slack_oauth_client_id, slack_oauth_client_secret, datatable, teams_team_guid, auto_invite, error_handler, success_handler, public_app_execution_limit_per_minute, error_handler_fallback_to_instance_alerts, guest_access_enabled, guest_jwt_public_key, guest_jwt_jwks_url, add_admins_and_developers_to_forks FROM workspace_settings WHERE workspace_id = $2", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Varchar", - "Text" - ] - }, - "nullable": [] - }, - "hash": "eefa0588a6a927fd9b3f65e1df652fb2b4cf7983049d2c490940df360c7e2b30" -} diff --git a/backend/.sqlx/query-ef98b0fab16d3b19997fd639b74a895cce1e8ae010307431e6787d20c7b25780.json b/backend/.sqlx/query-ef98b0fab16d3b19997fd639b74a895cce1e8ae010307431e6787d20c7b25780.json new file mode 100644 index 0000000000..3b89a91024 --- /dev/null +++ b/backend/.sqlx/query-ef98b0fab16d3b19997fd639b74a895cce1e8ae010307431e6787d20c7b25780.json @@ -0,0 +1,41 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT flow_version.id, flow_version.created_at, flow_version.created_by, deployment_metadata.deployment_msg FROM flow_version \n LEFT JOIN deployment_metadata ON flow_version.id = deployment_metadata.flow_version\n WHERE flow_version.path = $1 AND flow_version.workspace_id = $2 \n ORDER BY flow_version.created_at DESC", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Int8" + }, + { + "ordinal": 1, + "name": "created_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 2, + "name": "created_by", + "type_info": "Varchar" + }, + { + "ordinal": 3, + "name": "deployment_msg", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false, + false, + false, + true + ] + }, + "hash": "ef98b0fab16d3b19997fd639b74a895cce1e8ae010307431e6787d20c7b25780" +} diff --git a/backend/.sqlx/query-a2122e8520268919e2ddc85ef46b5f9322229bb2a1e104b6eabaf2a697c2776a.json b/backend/.sqlx/query-f3cd54017710536747b8aca797a8b983774ee2127dceb790181ed91b987f44ff.json similarity index 85% rename from backend/.sqlx/query-a2122e8520268919e2ddc85ef46b5f9322229bb2a1e104b6eabaf2a697c2776a.json rename to backend/.sqlx/query-f3cd54017710536747b8aca797a8b983774ee2127dceb790181ed91b987f44ff.json index 0f43afed39..6964dfa8d6 100644 --- a/backend/.sqlx/query-a2122e8520268919e2ddc85ef46b5f9322229bb2a1e104b6eabaf2a697c2776a.json +++ b/backend/.sqlx/query-f3cd54017710536747b8aca797a8b983774ee2127dceb790181ed91b987f44ff.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n SELECT\n nt.external_id,\n nt.workspace_id,\n nt.service_name AS \"service_name!: ServiceName\",\n nt.script_path,\n nt.is_flow,\n nt.webhook_token_hash,\n nt.service_config,\n nt.error,\n nt.created_at,\n nt.updated_at,\n nt.summary,\n nt.enabled\n FROM\n native_trigger nt\n WHERE\n nt.workspace_id = $1 AND\n nt.service_name = $2 AND\n ($5::text IS NULL OR nt.script_path = $5) AND\n ($6::bool IS NULL OR nt.is_flow = $6) AND\n (\n (nt.is_flow = false AND EXISTS (\n SELECT 1 FROM script s\n WHERE s.workspace_id = nt.workspace_id\n AND s.path = nt.script_path\n ))\n OR\n (nt.is_flow = true AND EXISTS (\n SELECT 1 FROM flow f\n WHERE f.workspace_id = nt.workspace_id\n AND f.path = nt.script_path\n ))\n )\n LIMIT $3\n OFFSET $4\n ", + "query": "\n SELECT\n nt.external_id,\n nt.workspace_id,\n nt.service_name AS \"service_name!: ServiceName\",\n nt.script_path,\n nt.is_flow,\n nt.webhook_token_hash,\n nt.service_config,\n nt.error,\n nt.created_at,\n nt.updated_at,\n nt.summary,\n nt.enabled\n FROM\n native_trigger nt\n WHERE\n nt.workspace_id = $1 AND\n nt.service_name = $2 AND\n ($5::text IS NULL OR nt.script_path = $5) AND\n ($6::bool IS NULL OR nt.is_flow = $6) AND\n (\n (nt.is_flow = false AND EXISTS (\n SELECT 1 FROM script s\n WHERE s.workspace_id = nt.workspace_id\n AND s.path = nt.script_path\n ))\n OR\n (nt.is_flow = true AND EXISTS (\n SELECT 1 FROM flow f\n WHERE f.workspace_id = nt.workspace_id\n AND f.path = nt.script_path\n ))\n ) AND\n ( $7\n OR nt.script_path = ANY($8)\n OR EXISTS ( SELECT 1 FROM unnest($9::text[]) AS pfx\n WHERE nt.script_path = pfx\n OR left(nt.script_path, length(pfx) + 1) = pfx || '/' ) )\n LIMIT $3\n OFFSET $4\n ", "describe": { "columns": [ { @@ -93,7 +93,10 @@ "Int8", "Int8", "Text", - "Bool" + "Bool", + "Bool", + "TextArray", + "TextArray" ] }, "nullable": [ @@ -111,5 +114,5 @@ false ] }, - "hash": "a2122e8520268919e2ddc85ef46b5f9322229bb2a1e104b6eabaf2a697c2776a" + "hash": "f3cd54017710536747b8aca797a8b983774ee2127dceb790181ed91b987f44ff" } diff --git a/backend/.sqlx/query-f7ece5036ad92e485b5e15a70e5052b42aaf5c87e5be6333b2df67a81990c8a6.json b/backend/.sqlx/query-f7ece5036ad92e485b5e15a70e5052b42aaf5c87e5be6333b2df67a81990c8a6.json new file mode 100644 index 0000000000..1be51feb85 --- /dev/null +++ b/backend/.sqlx/query-f7ece5036ad92e485b5e15a70e5052b42aaf5c87e5be6333b2df67a81990c8a6.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE workspace_settings ws\n SET datatable = (\n SELECT jsonb_set(ws.datatable, '{datatables}', jsonb_object_agg(\n dt.key,\n (SELECT CASE WHEN r.v->'governed_by'->>'workspace_id' = $1\n AND r.v->'governed_by'->>'datatable' = $2\n THEN jsonb_set(r.v, '{governed_by,datatable}', to_jsonb($3::text))\n ELSE r.v END\n FROM (SELECT CASE WHEN dt.value->'reference'->>'workspace_id' = $1\n AND dt.value->'reference'->>'datatable' = $2\n THEN jsonb_set(dt.value, '{reference,datatable}', to_jsonb($3::text))\n ELSE dt.value END AS v) r)\n ))\n FROM jsonb_each(ws.datatable->'datatables') dt\n )\n WHERE EXISTS (\n SELECT 1 FROM jsonb_each(COALESCE(ws.datatable->'datatables', '{}'::jsonb)) d,\n LATERAL (VALUES ('reference'), ('governed_by')) k(link)\n WHERE d.value->k.link->>'workspace_id' = $1\n AND d.value->k.link->>'datatable' = $2\n )", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "f7ece5036ad92e485b5e15a70e5052b42aaf5c87e5be6333b2df67a81990c8a6" +} diff --git a/backend/.sqlx/query-f9692bba979c2b5a22e65106a330a0c0827dea6115daf9d006f8ccda0d041d76.json b/backend/.sqlx/query-f9692bba979c2b5a22e65106a330a0c0827dea6115daf9d006f8ccda0d041d76.json new file mode 100644 index 0000000000..c3e7b0168d --- /dev/null +++ b/backend/.sqlx/query-f9692bba979c2b5a22e65106a330a0c0827dea6115daf9d006f8ccda0d041d76.json @@ -0,0 +1,20 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT count(*) FROM remote_deploy_token WHERE email = 'test2@windmill.dev'", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "count", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + null + ] + }, + "hash": "f9692bba979c2b5a22e65106a330a0c0827dea6115daf9d006f8ccda0d041d76" +} diff --git a/backend/.sqlx/query-fc2081d9077c743bc3978ae1905db558549b2356fc6ac8f0557aa40e1345ff46.json b/backend/.sqlx/query-fc2081d9077c743bc3978ae1905db558549b2356fc6ac8f0557aa40e1345ff46.json new file mode 100644 index 0000000000..27cb37b876 --- /dev/null +++ b/backend/.sqlx/query-fc2081d9077c743bc3978ae1905db558549b2356fc6ac8f0557aa40e1345ff46.json @@ -0,0 +1,31 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT d.email IS NULL as \"legacy!\", COALESCE(u.username, p.username) as username\n FROM draft d\n LEFT JOIN usr u ON u.workspace_id = d.workspace_id AND u.email = d.email\n LEFT JOIN password p ON p.email = d.email AND p.super_admin = true\n WHERE d.workspace_id = $1 AND d.path = $2 AND d.typ::text = ANY($3::text[])\n -- A row a move already brought here from the very path being renamed is this\n -- rename's own draft, not an item in its way: the record says which owners\n -- it moved (all of them for an item move, one for a draft-only move). Any\n -- other row is a second item and still collides.\n AND NOT EXISTS (\n SELECT 1 FROM draft_move m\n WHERE m.workspace_id = $1 AND m.typ::text = ANY($3::text[])\n AND m.old_path = $4 AND m.new_path = $2\n AND (m.email IS NULL OR m.email IS NOT DISTINCT FROM d.email)\n )\n ORDER BY 2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "legacy!", + "type_info": "Bool" + }, + { + "ordinal": 1, + "name": "username", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "TextArray", + "Text" + ] + }, + "nullable": [ + null, + null + ] + }, + "hash": "fc2081d9077c743bc3978ae1905db558549b2356fc6ac8f0557aa40e1345ff46" +} diff --git a/backend/AGENTS.md b/backend/AGENTS.md index 7bd84dfca3..750e0594e1 100644 --- a/backend/AGENTS.md +++ b/backend/AGENTS.md @@ -23,7 +23,7 @@ ## Cargo features & running the dev backend The dev backend runs under `cargo watch` and is launched by default with **only -`--features quickjs`** (see the tmux backend pane). That baseline compiles fast but +`--features quickjs`** (see the backend pane). That baseline compiles fast but **deliberately omits most functionality** — notably S3/object storage, the S3 proxy, all EE code, MCP, and every non-JS language runtime. A running server never gains a feature you didn't compile in: feature-gated routes 404 or return a `"requires "` stub. So if @@ -33,22 +33,30 @@ MUST **restart the backend with the appropriate features** for what you're worki ### Restarting the dev backend with the right features Restart in the **same pane**, so the relaunch inherits that pane's `DATABASE_URL`, `BACKEND_PORT` -and the rest of `runtime.env`. Scope every kill to this worktree — **never** +and the rest of the worktree environment. Scope every kill to this worktree — **never** `pkill -f target/debug/windmill`, which kills every sibling worktree's backend. -1. Find the backend pane by what it is running, not by index. The index depends on the webmux - profile: pane 1 is the backend under `full`, but the *frontend* under `frontendOnly`. +1. Find the backend pane by where it is running, not by index. Under herdr the two server panes + sit in the worktree's `backend/` and `frontend/`, so match on `cwd`: + + ```bash + herdr pane list --workspace "$HERDR_WORKSPACE_ID" + ``` + + Under webmux they are tmux panes, and the index depends on the profile: pane 1 is the backend + under `full`, but the *frontend* under `frontendOnly`. ```bash WIN=$(tmux display-message -p -t "$TMUX_PANE" '#{window_id}') tmux list-panes -t "$WIN" -F '#{pane_index} #{pane_current_command} #{pane_pid}' ``` -2. Read the feature set it is **actually** running. `CARGO_FEATURES` in `runtime.env` is only what - the pane started with, and goes stale the first time anyone restarts by hand: +2. Read the feature set it is **actually** running. The `CARGO_FEATURES` the worktree was created + with is only what the pane started with, and goes stale the first time anyone restarts by hand: ```bash - ps --ppid -o args= + herdr pane process-info --pane # foreground_processes[].argv + ps --ppid -o args= # under webmux # /home/hugo/.cargo/bin/cargo-watch watch -x run --features quickjs ``` @@ -56,22 +64,25 @@ and the rest of `runtime.env`. Scope every kill to this worktree — **never** explicitly: ```bash - tmux send-keys -t "$WIN." C-c - tmux send-keys -t "$WIN." 'PORT=$BACKEND_PORT cargo watch -x "run --features quickjs,private,parquet"' Enter + herdr pane send-keys C-c + herdr pane run 'PORT=$BACKEND_PORT cargo watch -x "run --features quickjs,private,parquet"' + # under webmux: tmux send-keys -t "$WIN." C-c, then the same command line with Enter ``` Carry over every feature the old command had unless you mean to drop one — rebuilding the list from memory is how a backend silently loses `quickjs`. -4. Persist the new set so a recreated pane starts with it: set `CARGO_FEATURES` in the - worktree's `.env.local`, which `scripts/post-create.sh` writes and webmux reads. Do **not** - edit `runtime.env` for this — webmux regenerates it from metadata and `.env.local` every time - the worktree is opened, so an edit there is lost on the next reopen. Either way the change - only affects a future pane; step 3 is what takes effect now. +4. Persist the new set so a recreated pane starts with it: set `CARGO_FEATURES` in the worktree's + `.env.local`, which the worktree hooks write. A herdr pane sources that file directly, so the + next pane picks the value up. A webmux pane sources `webmux/runtime.env` instead, which webmux + regenerates from metadata and `.env.local` when the worktree is *opened* — so the value lands + on the next reopen, not the next pane, and editing `runtime.env` by hand is lost at that same + reopen. Either way the change only affects a future pane; step 3 is what takes effect now. -5. Re-capture the pane until `health check completed` appears before hitting the API. A cold - rebuild takes ~60s, and the previous run's success line is still in the scrollback, so a - capture taken too early reads as ready when it isn't. +5. Re-read the pane (`herdr pane read --source recent-unwrapped`, or `tmux + capture-pane`) until `health check completed` appears before hitting the API. A cold rebuild + takes ~60s, and the previous run's success line is still in the scrollback, so a capture taken + too early reads as ready when it isn't. cargo-watch only re-runs on a file change, so after an idle/failed run `touch README.md` (from `backend/`, where the watch runs) is a cheap retrigger (touching a `.rs` forces a full rebuild). diff --git a/backend/Cargo.lock b/backend/Cargo.lock index ac121c6637..e0bfce842b 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -588,15 +588,15 @@ dependencies = [ [[package]] name = "astral-tokio-tar" -version = "0.5.6" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec179a06c1769b1e42e1e2cbe74c7dcdb3d6383c838454d063eaac5bbb7ebbe5" +checksum = "b18457efd137254e016bbde5e1d88df61c4e1a5ae2223746e56123bac6af2463" dependencies = [ - "filetime", "futures-core", "libc", "portable-atomic", "rustc-hash 2.1.3", + "rustix 1.1.5", "tokio", "tokio-stream", "xattr", @@ -928,9 +928,9 @@ checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" [[package]] name = "aws-config" -version = "1.8.3" +version = "1.8.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0baa720ebadea158c5bda642ac444a2af0cdf7bb66b46d1e4533de5d1f449d0" +checksum = "8a8fc176d53d6fe85017f230405e3255cedb4a02221cb55ed6d76dccbbb099b2" dependencies = [ "aws-credential-types", "aws-runtime", @@ -938,8 +938,8 @@ dependencies = [ "aws-sdk-ssooidc", "aws-sdk-sts", "aws-smithy-async", - "aws-smithy-http 0.62.6", - "aws-smithy-json 0.61.9", + "aws-smithy-http 0.63.4", + "aws-smithy-json 0.62.4", "aws-smithy-runtime", "aws-smithy-runtime-api", "aws-smithy-types", @@ -1070,15 +1070,16 @@ dependencies = [ [[package]] name = "aws-sdk-config" -version = "1.68.0" +version = "1.101.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7de8bcff88ff2948bf2154204b7eba0ea30698c1c23770c4a917505fb9fcbd5" +checksum = "0360cf8f8ab0e328f89b5c60bdcc6203ebce83b8a018c05ed177c4f32f819aff" dependencies = [ "aws-credential-types", "aws-runtime", "aws-smithy-async", - "aws-smithy-http 0.62.6", - "aws-smithy-json 0.61.9", + "aws-smithy-http 0.63.4", + "aws-smithy-json 0.62.4", + "aws-smithy-observability", "aws-smithy-runtime", "aws-smithy-runtime-api", "aws-smithy-types", @@ -1086,7 +1087,7 @@ dependencies = [ "bytes", "fastrand 2.5.0", "http 0.2.12", - "once_cell", + "http 1.5.0", "regex-lite", "tracing", ] @@ -1144,15 +1145,16 @@ dependencies = [ [[package]] name = "aws-sdk-sqs" -version = "1.77.0" +version = "1.94.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32a6dcd63dffae1f7872df42ce9f1f0f785f8c9a3d3e880ef6c171d0415439df" +checksum = "0894909314920ab2c907110e7f91b9f7a6c826bdc830ccf36079c69af2b82335" dependencies = [ "aws-credential-types", "aws-runtime", "aws-smithy-async", - "aws-smithy-http 0.62.6", - "aws-smithy-json 0.61.9", + "aws-smithy-http 0.63.4", + "aws-smithy-json 0.62.4", + "aws-smithy-observability", "aws-smithy-runtime", "aws-smithy-runtime-api", "aws-smithy-types", @@ -1160,6 +1162,7 @@ dependencies = [ "bytes", "fastrand 2.5.0", "http 0.2.12", + "http 1.5.0", "regex-lite", "tracing", ] @@ -1214,15 +1217,16 @@ dependencies = [ [[package]] name = "aws-sdk-sts" -version = "1.79.0" +version = "1.98.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "317377afba3498fca4948c5d32b399ef9a5ad35561a1e8a6f2ac7273dabf802d" +checksum = "89c4f19655ab0856375e169865c91264de965bd74c407c7f1e403184b1049409" dependencies = [ "aws-credential-types", "aws-runtime", "aws-smithy-async", - "aws-smithy-http 0.62.6", - "aws-smithy-json 0.61.9", + "aws-smithy-http 0.63.4", + "aws-smithy-json 0.62.4", + "aws-smithy-observability", "aws-smithy-query", "aws-smithy-runtime", "aws-smithy-runtime-api", @@ -1231,6 +1235,7 @@ dependencies = [ "aws-types", "fastrand 2.5.0", "http 0.2.12", + "http 1.5.0", "regex-lite", "tracing", ] @@ -1340,7 +1345,7 @@ dependencies = [ "hyper 0.14.32", "hyper 1.11.1", "hyper-rustls 0.24.2", - "hyper-rustls 0.27.9", + "hyper-rustls 0.27.10", "hyper-util", "pin-project-lite", "rustls 0.21.12", @@ -2221,7 +2226,7 @@ dependencies = [ "memmap2", "num-traits", "num_cpus", - "rand 0.9.0", + "rand 0.9.5", "rand_distr", "rayon", "safetensors", @@ -2257,7 +2262,7 @@ dependencies = [ "candle-nn", "fancy-regex 0.17.0", "num-traits", - "rand 0.9.0", + "rand 0.9.5", "rayon", "serde", "serde_json", @@ -3996,7 +4001,7 @@ dependencies = [ "http 1.5.0", "http-body-util", "hyper 1.11.1", - "hyper-rustls 0.27.9", + "hyper-rustls 0.27.10", "hyper-util", "ipnet", "percent-encoding", @@ -4199,7 +4204,7 @@ dependencies = [ "deno_tls", "http-body-util", "hyper 1.11.1", - "hyper-rustls 0.27.9", + "hyper-rustls 0.27.10", "hyper-util", "log", "once_cell", @@ -5101,7 +5106,7 @@ checksum = "719a903cc23e4a89e87962c2a80fdb45cdaad0983a89bd150bb57b4c8571a7d5" dependencies = [ "half", "num-traits", - "rand 0.9.0", + "rand 0.9.5", "rand_distr", ] @@ -5833,7 +5838,7 @@ dependencies = [ "cfg-if", "crunchy", "num-traits", - "rand 0.9.0", + "rand 0.9.5", "rand_distr", "zerocopy", ] @@ -5975,7 +5980,7 @@ dependencies = [ "log", "native-tls", "num_cpus", - "rand 0.9.0", + "rand 0.9.5", "reqwest 0.12.28", "serde", "serde_json", @@ -6002,7 +6007,7 @@ dependencies = [ "idna", "ipnet", "once_cell", - "rand 0.9.0", + "rand 0.9.5", "serde", "thiserror 2.0.20", "tinyvec", @@ -6024,7 +6029,7 @@ dependencies = [ "moka", "once_cell", "parking_lot", - "rand 0.9.0", + "rand 0.9.5", "resolv-conf", "serde", "smallvec", @@ -6258,7 +6263,7 @@ dependencies = [ "headers", "http 1.5.0", "hyper 1.11.1", - "hyper-rustls 0.27.9", + "hyper-rustls 0.27.10", "hyper-util", "pin-project-lite", "tokio", @@ -6317,9 +6322,9 @@ dependencies = [ [[package]] name = "hyper-rustls" -version = "0.27.9" +version = "0.27.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33ca68d021ef39cf6463ab54c1d0f5daf03377b70561305bb89a8f83aab66e0f" +checksum = "dfa8e654703247911e29c23fbeaa261834bd9bb74efba2f9acddc37bfb127f53" dependencies = [ "http 1.5.0", "hyper 1.11.1", @@ -6949,13 +6954,14 @@ dependencies = [ [[package]] name = "jsonwebtoken" -version = "8.3.0" +version = "9.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6971da4d9c3aa03c3d8f3ff0f4155b534aad021292003895a469716b2a230378" +checksum = "5a87cc7a48537badeae96744432de36f4be2b4a34a05a5ef32e9dd8a1c169dde" dependencies = [ - "base64 0.21.7", - "pem 1.1.1", - "ring 0.16.20", + "base64 0.22.1", + "js-sys", + "pem", + "ring 0.17.14", "serde", "serde_json", "simple_asn1", @@ -6963,17 +6969,20 @@ dependencies = [ [[package]] name = "jsonwebtoken" -version = "9.3.1" +version = "10.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a87cc7a48537badeae96744432de36f4be2b4a34a05a5ef32e9dd8a1c169dde" +checksum = "eba32bfb4ffdeaca3e34431072faf01745c9b26d25504aa7a6cf5684334fc4fc" dependencies = [ + "aws-lc-rs", "base64 0.22.1", + "getrandom 0.2.17", "js-sys", - "pem 3.0.6", - "ring 0.17.14", + "pem", "serde", "serde_json", + "signature", "simple_asn1", + "zeroize", ] [[package]] @@ -7062,13 +7071,13 @@ dependencies = [ "http-body-util", "hyper 1.11.1", "hyper-http-proxy", - "hyper-rustls 0.27.9", + "hyper-rustls 0.27.10", "hyper-timeout", "hyper-util", "jsonpath-rust", "k8s-openapi", "kube-core", - "pem 3.0.6", + "pem", "rustls 0.23.35", "secrecy", "serde", @@ -7308,9 +7317,9 @@ dependencies = [ [[package]] name = "libredox" -version = "0.1.24" +version = "0.1.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6480ccc157a1389bb2e4891b24751b0f798ba640d22386f23143fbcc89da195a" +checksum = "61ff90caf6077a803a240f62fdbe88645a890bbca49ef8174c3cb0404362171d" dependencies = [ "bitflags 2.13.2", "libc", @@ -7932,9 +7941,9 @@ dependencies = [ "lru 0.18.4", "mysql_common", "native-tls", - "pem 3.0.6", + "pem", "percent-encoding", - "rand 0.10.2", + "rand 0.10.3", "serde", "socket2 0.6.5", "thiserror 2.0.20", @@ -8440,7 +8449,7 @@ dependencies = [ "parking_lot", "percent-encoding", "quick-xml", - "rand 0.9.0", + "rand 0.9.5", "reqwest 0.12.28", "ring 0.17.14", "rustls-pemfile 2.2.0", @@ -8809,7 +8818,7 @@ dependencies = [ "futures-util", "opentelemetry 0.30.0", "percent-encoding", - "rand 0.9.0", + "rand 0.9.5", "serde_json", "thiserror 2.0.20", "tokio", @@ -8911,7 +8920,7 @@ dependencies = [ "hmac", "pkcs12", "pkcs5", - "rand 0.9.0", + "rand 0.9.5", "rc2", "sha1", "sha2 0.10.9", @@ -9075,15 +9084,6 @@ dependencies = [ "hmac", ] -[[package]] -name = "pem" -version = "1.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8835c273a76a90455d7344889b0964598e3316e2a79ede8e36f16bdcf2228b8" -dependencies = [ - "base64 0.13.1", -] - [[package]] name = "pem" version = "3.0.6" @@ -9123,9 +9123,9 @@ checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" [[package]] name = "pest" -version = "2.9.1" +version = "2.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d45aeb61b4bf818e12d4205f2466f8c4748f85f4fce0146d1c03d69d753f0ad" +checksum = "45d3aca230fad2e6f6317ca0a72724338c4960cb97168a85cdee66df4a9a21a8" dependencies = [ "memchr", "ucd-trie", @@ -9133,9 +9133,9 @@ dependencies = [ [[package]] name = "pest_derive" -version = "2.9.1" +version = "2.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89cc5a242e25ed4e7704d0be240f2cfbe20a8c27e7e252d94835be93d92dc39f" +checksum = "284b60557f2c4a2e72ad3f2d34d42685a2fa4a6a61d0d2a10c0ae2a5e916c2cf" dependencies = [ "pest", "pest_generator", @@ -9143,9 +9143,9 @@ dependencies = [ [[package]] name = "pest_generator" -version = "2.9.1" +version = "2.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7abf21475cc3820fe4b2ca2dc2142902f67a02189f3b5b3a229f4febc01a43e5" +checksum = "1d9d1f08a115309ee99268cf85e5228e0e56aa9caf8841ec12866b6be07c3109" dependencies = [ "pest", "pest_meta", @@ -9156,9 +9156,9 @@ dependencies = [ [[package]] name = "pest_meta" -version = "2.9.1" +version = "2.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "adba4db388f687393c18c51348d44a41d870ca9df71a2c98172ea3035dc6936e" +checksum = "ed93ba1a9ffcca32130a5188701c81c0c49cf00d4b7c5007d5148951d743adcb" dependencies = [ "pest", ] @@ -9504,7 +9504,7 @@ dependencies = [ "hmac", "md-5 0.10.6", "memchr", - "rand 0.9.0", + "rand 0.9.5", "sha2 0.10.9", "stringprep", ] @@ -9879,7 +9879,7 @@ dependencies = [ "bytes", "getrandom 0.4.3", "lru-slab", - "rand 0.10.2", + "rand 0.10.3", "rand_pcg", "ring 0.17.14", "rustc-hash 2.1.3", @@ -9959,20 +9959,19 @@ dependencies = [ [[package]] name = "rand" -version = "0.9.0" +version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3779b94aeb87e8bd4e834cee3650289ee9e0d5677f976ecdb6d219e5f4f6cd94" +checksum = "b9ef1d0d795eb7d84685bca4f72f3649f064e6641543d3a8c415898726a57b41" dependencies = [ "rand_chacha 0.9.0", "rand_core 0.9.5", - "zerocopy", ] [[package]] name = "rand" -version = "0.10.2" +version = "0.10.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7f5fa3a058cd35567ef9bfa5e75732bee0f9e4c55fa90477bef2dfcdbc4be80" +checksum = "65c9fb96cbc91e3478eaae79a69fcd3f1ae4ad052e471fe6732fff548984b4af" dependencies = [ "chacha20", "getrandom 0.4.3", @@ -10049,7 +10048,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a8615d50dcf34fa31f7ab52692afec947c4dd0ab803cc87cb3b0b4570ff7463" dependencies = [ "num-traits", - "rand 0.9.0", + "rand 0.9.5", ] [[package]] @@ -10125,7 +10124,7 @@ version = "0.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "75e669e5202259b5314d1ea5397316ad400819437857b90861765f24c4cf80a2" dependencies = [ - "pem 3.0.6", + "pem", "ring 0.17.14", "rustls-pki-types", "time", @@ -10339,7 +10338,7 @@ dependencies = [ "http-body 1.1.0", "http-body-util", "hyper 1.11.1", - "hyper-rustls 0.27.9", + "hyper-rustls 0.27.10", "hyper-tls", "hyper-util", "js-sys", @@ -10387,7 +10386,7 @@ dependencies = [ "http-body 1.1.0", "http-body-util", "hyper 1.11.1", - "hyper-rustls 0.27.9", + "hyper-rustls 0.27.10", "hyper-util", "js-sys", "log", @@ -10464,7 +10463,7 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc05fbf560421a0357a750cbe78c7ca19d4923918490daabba313d5dbc871e47" dependencies = [ - "rand 0.10.2", + "rand 0.10.3", ] [[package]] @@ -10552,7 +10551,7 @@ dependencies = [ "oauth2", "pastey", "pin-project-lite", - "rand 0.10.2", + "rand 0.10.3", "reqwest 0.13.5", "rmcp-macros", "schemars 1.2.2", @@ -11114,7 +11113,7 @@ dependencies = [ "openssl-sys", "pkg-config", "quick-xml", - "rand 0.9.0", + "rand 0.9.5", "serde", "thiserror 2.0.20", "url", @@ -13397,7 +13396,7 @@ dependencies = [ "pin-project-lite", "postgres-protocol", "postgres-types", - "rand 0.9.0", + "rand 0.9.5", "socket2 0.6.5", "tokio", "tokio-util", @@ -14089,7 +14088,7 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "470dbf6591da1b39d43c14523b2b469c86879a53e8b758c8e090a470fe7b1fbe" dependencies = [ - "rand 0.9.0", + "rand 0.9.5", "uuid", "web-time", ] @@ -14172,9 +14171,9 @@ checksum = "5c1cb5db39152898a79168971543b1cb5020dff7fe43c8dc468b0885f5e29df5" [[package]] name = "unicode-id-start" -version = "1.4.0" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81b79ad29b5e19de4260020f8919b443b2ef0277d242ce532ec7b7a2cc8b6007" +checksum = "0825d2e09dcae814a4273c1e1435979d414ede25b2ee235b1545da6682483765" [[package]] name = "unicode-ident" @@ -14793,7 +14792,7 @@ dependencies = [ [[package]] name = "windmill" -version = "1.815.0" +version = "1.817.0" dependencies = [ "anyhow", "async-nats", @@ -14810,13 +14809,13 @@ dependencies = [ "git-version", "hex", "hmac", - "jsonwebtoken 8.3.0", + "jsonwebtoken 10.4.0", "lazy_static", "once_cell", "opentelemetry 0.30.0", "opentelemetry_sdk 0.30.0", "prometheus", - "rand 0.9.0", + "rand 0.9.5", "rdkafka", "reqwest 0.13.5", "rumqttc", @@ -14881,7 +14880,7 @@ dependencies = [ [[package]] name = "windmill-ai" -version = "1.815.0" +version = "1.817.0" dependencies = [ "async-stream", "async-trait", @@ -14915,7 +14914,7 @@ dependencies = [ [[package]] name = "windmill-alerting" -version = "1.815.0" +version = "1.817.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14928,7 +14927,7 @@ dependencies = [ [[package]] name = "windmill-api" -version = "1.815.0" +version = "1.817.0" dependencies = [ "anyhow", "argon2", @@ -14961,7 +14960,7 @@ dependencies = [ "hyper 1.11.1", "indexmap 2.14.2", "itertools 0.14.0", - "jsonwebtoken 8.3.0", + "jsonwebtoken 10.4.0", "lazy_static", "libxml", "magic-crypt", @@ -14976,7 +14975,7 @@ dependencies = [ "prometheus", "quick-xml", "quick_cache", - "rand 0.9.0", + "rand 0.9.5", "regex", "reqwest 0.13.5", "rsa", @@ -15068,7 +15067,7 @@ dependencies = [ [[package]] name = "windmill-api-agent-workers" -version = "1.815.0" +version = "1.817.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15091,7 +15090,7 @@ dependencies = [ [[package]] name = "windmill-api-assets" -version = "1.815.0" +version = "1.817.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15108,14 +15107,14 @@ dependencies = [ [[package]] name = "windmill-api-auth" -version = "1.815.0" +version = "1.817.0" dependencies = [ "anyhow", "axum 0.8.9", "chrono", "http 1.5.0", "itertools 0.14.0", - "jsonwebtoken 8.3.0", + "jsonwebtoken 10.4.0", "lazy_static", "quick_cache", "reqwest 0.13.5", @@ -15134,7 +15133,7 @@ dependencies = [ [[package]] name = "windmill-api-client" -version = "1.815.0" +version = "1.817.0" dependencies = [ "reqwest 0.12.28", "serde", @@ -15144,7 +15143,7 @@ dependencies = [ [[package]] name = "windmill-api-configs" -version = "1.815.0" +version = "1.817.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15161,7 +15160,7 @@ dependencies = [ [[package]] name = "windmill-api-debug" -version = "1.815.0" +version = "1.817.0" dependencies = [ "axum 0.8.9", "base64 0.22.1", @@ -15183,7 +15182,7 @@ dependencies = [ [[package]] name = "windmill-api-embeddings" -version = "1.815.0" +version = "1.817.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -15206,7 +15205,7 @@ dependencies = [ [[package]] name = "windmill-api-flow-conversations" -version = "1.815.0" +version = "1.817.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15222,7 +15221,7 @@ dependencies = [ [[package]] name = "windmill-api-flows" -version = "1.815.0" +version = "1.817.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15244,7 +15243,7 @@ dependencies = [ [[package]] name = "windmill-api-groups" -version = "1.815.0" +version = "1.817.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15266,7 +15265,7 @@ dependencies = [ [[package]] name = "windmill-api-inputs" -version = "1.815.0" +version = "1.817.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15280,7 +15279,7 @@ dependencies = [ [[package]] name = "windmill-api-integration-tests" -version = "1.815.0" +version = "1.817.0" dependencies = [ "anyhow", "async-nats", @@ -15292,7 +15291,7 @@ dependencies = [ "futures", "hex", "hmac", - "rand 0.9.0", + "rand 0.9.5", "rdkafka", "reqwest 0.13.5", "rmcp", @@ -15315,7 +15314,7 @@ dependencies = [ [[package]] name = "windmill-api-jobs" -version = "1.815.0" +version = "1.817.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -15340,7 +15339,7 @@ dependencies = [ [[package]] name = "windmill-api-npm-proxy" -version = "1.815.0" +version = "1.817.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15368,7 +15367,7 @@ dependencies = [ [[package]] name = "windmill-api-openapi" -version = "1.815.0" +version = "1.817.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -15390,7 +15389,7 @@ dependencies = [ [[package]] name = "windmill-api-schedule" -version = "1.815.0" +version = "1.817.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15410,7 +15409,7 @@ dependencies = [ [[package]] name = "windmill-api-scripts" -version = "1.815.0" +version = "1.817.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15448,7 +15447,7 @@ dependencies = [ [[package]] name = "windmill-api-settings" -version = "1.815.0" +version = "1.817.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -15477,7 +15476,7 @@ dependencies = [ [[package]] name = "windmill-api-sse" -version = "1.815.0" +version = "1.817.0" dependencies = [ "lazy_static", "serde", @@ -15489,7 +15488,7 @@ dependencies = [ [[package]] name = "windmill-api-users" -version = "1.815.0" +version = "1.817.0" dependencies = [ "argon2", "axum 0.8.9", @@ -15513,7 +15512,7 @@ dependencies = [ [[package]] name = "windmill-api-workers" -version = "1.815.0" +version = "1.817.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15527,10 +15526,11 @@ dependencies = [ [[package]] name = "windmill-api-workspaces" -version = "1.815.0" +version = "1.817.0" dependencies = [ "axum 0.8.9", "chrono", + "constant_time_eq 0.3.1", "futures", "hex", "http 1.5.0", @@ -15538,6 +15538,7 @@ dependencies = [ "lazy_static", "magic-crypt", "regex", + "reqwest 0.13.5", "serde", "serde_json", "sha2 0.10.9", @@ -15546,6 +15547,7 @@ dependencies = [ "tokio", "tokio-postgres", "tracing", + "url", "uuid", "windmill-api-auth", "windmill-api-jobs", @@ -15562,7 +15564,7 @@ dependencies = [ [[package]] name = "windmill-audit" -version = "1.815.0" +version = "1.817.0" dependencies = [ "chrono", "lazy_static", @@ -15576,7 +15578,7 @@ dependencies = [ [[package]] name = "windmill-autoscaling" -version = "1.815.0" +version = "1.817.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -15595,7 +15597,7 @@ dependencies = [ [[package]] name = "windmill-common" -version = "1.815.0" +version = "1.817.0" dependencies = [ "aes-gcm", "aho-corasick", @@ -15635,7 +15637,7 @@ dependencies = [ "hyper 1.11.1", "indexmap 2.14.2", "itertools 0.14.0", - "jsonwebtoken 8.3.0", + "jsonwebtoken 10.4.0", "lazy_static", "magic-crypt", "mail-send", @@ -15655,7 +15657,7 @@ dependencies = [ "postgres-native-tls 0.5.3", "prometheus", "quick_cache", - "rand 0.9.0", + "rand 0.9.5", "regex", "reqwest 0.13.5", "reqwest-middleware", @@ -15702,7 +15704,7 @@ dependencies = [ [[package]] name = "windmill-dep-map" -version = "1.815.0" +version = "1.817.0" dependencies = [ "chrono", "futures", @@ -15722,7 +15724,7 @@ dependencies = [ [[package]] name = "windmill-git-sync" -version = "1.815.0" +version = "1.817.0" dependencies = [ "regex", "serde", @@ -15739,7 +15741,7 @@ dependencies = [ [[package]] name = "windmill-indexer" -version = "1.815.0" +version = "1.817.0" dependencies = [ "anyhow", "astral-tokio-tar", @@ -15766,7 +15768,7 @@ dependencies = [ [[package]] name = "windmill-jseval" -version = "1.815.0" +version = "1.817.0" dependencies = [ "anyhow", "futures", @@ -15783,7 +15785,7 @@ dependencies = [ [[package]] name = "windmill-macros" -version = "1.815.0" +version = "1.817.0" dependencies = [ "itertools 0.14.0", "lazy_static", @@ -15799,7 +15801,7 @@ dependencies = [ [[package]] name = "windmill-mcp" -version = "1.815.0" +version = "1.817.0" dependencies = [ "anyhow", "async-trait", @@ -15820,7 +15822,7 @@ dependencies = [ [[package]] name = "windmill-native-triggers" -version = "1.815.0" +version = "1.817.0" dependencies = [ "anyhow", "async-trait", @@ -15851,7 +15853,7 @@ dependencies = [ [[package]] name = "windmill-oauth" -version = "1.815.0" +version = "1.817.0" dependencies = [ "anyhow", "arc-swap", @@ -15876,7 +15878,7 @@ dependencies = [ [[package]] name = "windmill-object-store" -version = "1.815.0" +version = "1.817.0" dependencies = [ "anyhow", "async-stream", @@ -15911,7 +15913,7 @@ dependencies = [ [[package]] name = "windmill-operator" -version = "1.815.0" +version = "1.817.0" dependencies = [ "anyhow", "futures", @@ -15929,7 +15931,7 @@ dependencies = [ [[package]] name = "windmill-parser" -version = "1.815.0" +version = "1.817.0" dependencies = [ "convert_case 0.6.0", "serde", @@ -15938,7 +15940,7 @@ dependencies = [ [[package]] name = "windmill-parser-bash" -version = "1.815.0" +version = "1.817.0" dependencies = [ "anyhow", "lazy_static", @@ -15950,7 +15952,7 @@ dependencies = [ [[package]] name = "windmill-parser-csharp" -version = "1.815.0" +version = "1.817.0" dependencies = [ "anyhow", "serde_json", @@ -15962,7 +15964,7 @@ dependencies = [ [[package]] name = "windmill-parser-go" -version = "1.815.0" +version = "1.817.0" dependencies = [ "anyhow", "gosyn", @@ -15974,7 +15976,7 @@ dependencies = [ [[package]] name = "windmill-parser-graphql" -version = "1.815.0" +version = "1.817.0" dependencies = [ "anyhow", "lazy_static", @@ -15986,7 +15988,7 @@ dependencies = [ [[package]] name = "windmill-parser-java" -version = "1.815.0" +version = "1.817.0" dependencies = [ "anyhow", "serde_json", @@ -15998,7 +16000,7 @@ dependencies = [ [[package]] name = "windmill-parser-nu" -version = "1.815.0" +version = "1.817.0" dependencies = [ "anyhow", "nu-parser", @@ -16009,7 +16011,7 @@ dependencies = [ [[package]] name = "windmill-parser-php" -version = "1.815.0" +version = "1.817.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -16020,7 +16022,7 @@ dependencies = [ [[package]] name = "windmill-parser-py" -version = "1.815.0" +version = "1.817.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -16032,7 +16034,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-asset" -version = "1.815.0" +version = "1.817.0" dependencies = [ "anyhow", "rustpython-ast", @@ -16043,7 +16045,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-imports" -version = "1.815.0" +version = "1.817.0" dependencies = [ "anyhow", "async-recursion", @@ -16065,7 +16067,7 @@ dependencies = [ [[package]] name = "windmill-parser-r" -version = "1.815.0" +version = "1.817.0" dependencies = [ "anyhow", "serde_json", @@ -16077,7 +16079,7 @@ dependencies = [ [[package]] name = "windmill-parser-ruby" -version = "1.815.0" +version = "1.817.0" dependencies = [ "anyhow", "lazy_static", @@ -16091,7 +16093,7 @@ dependencies = [ [[package]] name = "windmill-parser-rust" -version = "1.815.0" +version = "1.817.0" dependencies = [ "anyhow", "convert_case 0.6.0", @@ -16108,7 +16110,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql" -version = "1.815.0" +version = "1.817.0" dependencies = [ "anyhow", "lazy_static", @@ -16121,7 +16123,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql-asset" -version = "1.815.0" +version = "1.817.0" dependencies = [ "anyhow", "serde", @@ -16133,7 +16135,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts" -version = "1.815.0" +version = "1.817.0" dependencies = [ "anyhow", "lazy_static", @@ -16151,7 +16153,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts-asset" -version = "1.815.0" +version = "1.817.0" dependencies = [ "anyhow", "serde-wasm-bindgen", @@ -16167,7 +16169,7 @@ dependencies = [ [[package]] name = "windmill-parser-wac" -version = "1.815.0" +version = "1.817.0" dependencies = [ "anyhow", "rustpython-ast", @@ -16183,7 +16185,7 @@ dependencies = [ [[package]] name = "windmill-parser-yaml" -version = "1.815.0" +version = "1.817.0" dependencies = [ "anyhow", "lazy_static", @@ -16197,7 +16199,7 @@ dependencies = [ [[package]] name = "windmill-queue" -version = "1.815.0" +version = "1.817.0" dependencies = [ "anyhow", "async-recursion", @@ -16216,7 +16218,7 @@ dependencies = [ "once_cell", "prometheus", "quick_cache", - "rand 0.9.0", + "rand 0.9.5", "regex", "reqwest 0.13.5", "serde", @@ -16236,7 +16238,7 @@ dependencies = [ [[package]] name = "windmill-runtime-nativets" -version = "1.815.0" +version = "1.817.0" dependencies = [ "anyhow", "const_format", @@ -16276,7 +16278,7 @@ dependencies = [ [[package]] name = "windmill-sql-datatype-parser-wasm" -version = "1.815.0" +version = "1.817.0" dependencies = [ "getrandom 0.3.4", "wasm-bindgen", @@ -16287,7 +16289,7 @@ dependencies = [ [[package]] name = "windmill-store" -version = "1.815.0" +version = "1.817.0" dependencies = [ "anyhow", "async-recursion", @@ -16322,7 +16324,7 @@ dependencies = [ [[package]] name = "windmill-test-utils" -version = "1.815.0" +version = "1.817.0" dependencies = [ "anyhow", "async-trait", @@ -16346,7 +16348,7 @@ dependencies = [ [[package]] name = "windmill-trigger" -version = "1.815.0" +version = "1.817.0" dependencies = [ "anyhow", "async-trait", @@ -16356,7 +16358,7 @@ dependencies = [ "hyper 1.11.1", "itertools 0.14.0", "lazy_static", - "rand 0.9.0", + "rand 0.9.5", "reqwest 0.13.5", "serde", "serde_json", @@ -16379,7 +16381,7 @@ dependencies = [ [[package]] name = "windmill-trigger-amqp" -version = "1.815.0" +version = "1.817.0" dependencies = [ "anyhow", "async-trait", @@ -16406,7 +16408,7 @@ dependencies = [ [[package]] name = "windmill-trigger-azure" -version = "1.815.0" +version = "1.817.0" dependencies = [ "anyhow", "async-trait", @@ -16420,7 +16422,7 @@ dependencies = [ "itertools 0.14.0", "lazy_static", "quick_cache", - "rand 0.9.0", + "rand 0.9.5", "reqwest 0.13.5", "serde", "serde_json", @@ -16439,7 +16441,7 @@ dependencies = [ [[package]] name = "windmill-trigger-email" -version = "1.815.0" +version = "1.817.0" dependencies = [ "anyhow", "async-trait", @@ -16459,7 +16461,7 @@ dependencies = [ [[package]] name = "windmill-trigger-gcp" -version = "1.815.0" +version = "1.817.0" dependencies = [ "anyhow", "async-trait", @@ -16471,7 +16473,7 @@ dependencies = [ "google-cloud-pubsub", "http 1.5.0", "itertools 0.14.0", - "jsonwebtoken 8.3.0", + "jsonwebtoken 10.4.0", "lazy_static", "quick_cache", "reqwest 0.13.5", @@ -16493,7 +16495,7 @@ dependencies = [ [[package]] name = "windmill-trigger-http" -version = "1.815.0" +version = "1.817.0" dependencies = [ "anyhow", "async-trait", @@ -16529,7 +16531,7 @@ dependencies = [ [[package]] name = "windmill-trigger-kafka" -version = "1.815.0" +version = "1.817.0" dependencies = [ "anyhow", "async-trait", @@ -16552,7 +16554,7 @@ dependencies = [ [[package]] name = "windmill-trigger-mqtt" -version = "1.815.0" +version = "1.817.0" dependencies = [ "anyhow", "async-trait", @@ -16576,7 +16578,7 @@ dependencies = [ [[package]] name = "windmill-trigger-nats" -version = "1.815.0" +version = "1.817.0" dependencies = [ "anyhow", "async-nats", @@ -16600,7 +16602,7 @@ dependencies = [ [[package]] name = "windmill-trigger-postgres" -version = "1.815.0" +version = "1.817.0" dependencies = [ "anyhow", "async-trait", @@ -16615,7 +16617,7 @@ dependencies = [ "pg_escape", "postgres-native-tls 0.5.2", "quick_cache", - "rand 0.9.0", + "rand 0.9.5", "rust_decimal", "serde", "serde_json", @@ -16635,7 +16637,7 @@ dependencies = [ [[package]] name = "windmill-trigger-sqs" -version = "1.815.0" +version = "1.817.0" dependencies = [ "anyhow", "async-trait", @@ -16663,7 +16665,7 @@ dependencies = [ [[package]] name = "windmill-trigger-websocket" -version = "1.815.0" +version = "1.817.0" dependencies = [ "anyhow", "async-trait", @@ -16688,14 +16690,14 @@ dependencies = [ [[package]] name = "windmill-types" -version = "1.815.0" +version = "1.817.0" dependencies = [ "anyhow", "bitflags 2.13.2", "chrono", "hex", "itertools 0.14.0", - "rand 0.9.0", + "rand 0.9.5", "serde", "serde_json", "sqlx", @@ -16707,7 +16709,7 @@ dependencies = [ [[package]] name = "windmill-worker" -version = "1.815.0" +version = "1.817.0" dependencies = [ "anyhow", "async-once-cell", @@ -16736,10 +16738,10 @@ dependencies = [ "hmac", "hudsucker", "hyper-http-proxy", - "hyper-rustls 0.27.9", + "hyper-rustls 0.27.10", "hyper-util", "itertools 0.14.0", - "jsonwebtoken 8.3.0", + "jsonwebtoken 10.4.0", "lazy_static", "libffi-sys", "libloading", @@ -16753,14 +16755,14 @@ dependencies = [ "opentelemetry-proto 0.30.0", "oracle", "parquet", - "pem 3.0.6", + "pem", "pep440_rs", "postgres-native-tls 0.5.3", "process-wrap", "prometheus", "prost", "quick_cache", - "rand 0.9.0", + "rand 0.9.5", "rcgen", "regex", "reqwest 0.13.5", @@ -16825,7 +16827,7 @@ dependencies = [ [[package]] name = "windmill-worker-volumes" -version = "1.815.0" +version = "1.817.0" dependencies = [ "bytes", "futures", diff --git a/backend/Cargo.toml b/backend/Cargo.toml index 258947cfb2..74da8a4244 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "windmill" -version = "1.815.0" +version = "1.817.0" authors.workspace = true edition.workspace = true @@ -88,7 +88,7 @@ members = [ exclude = ["./windmill-duckdb-ffi-internal", "./parsers/windmill-parser-wasm"] [workspace.package] -version = "1.815.0" +version = "1.817.0" authors = ["Ruben Fiszel "] edition = "2021" @@ -493,7 +493,7 @@ sql-builder = "^3" # behind a feature and changed `hash_password`), so a float breaks the build. argon2 = "0.6" quick_cache = "^0" -rand = "=0.9.0" +rand = "^0.9.3" rand_core = { version = "^0", features = ["std"] } arc-swap = "1" ed25519-dalek = { version = "2", features = ["rand_core"] } @@ -514,7 +514,7 @@ reqwest = { version = "^0.13", features = ["json", "stream", "gzip", "multipart" eventsource-stream = "0.2.3" time = "^0" serde_urlencoded = "^0" -astral-tokio-tar = "^0.5.6" +astral-tokio-tar = "^0.6.2" tempfile = "^3" x509-parser = "^0.16" tokio-util = { version = "=0.7.17", features = ["io"] } @@ -523,7 +523,7 @@ itertools = "^0.14.0" regex = "^1" semver = "^1" aws-sigv4 = "^1.3.4" -aws-sdk-config = "=1.68.0" +aws-sdk-config = "^1.94.0" aws-sdk-rds = "^1" async-trait = "0.1.88" @@ -637,7 +637,9 @@ libxml = { version = "=0.3.3" } samael = { git="https://github.com/njaremko/samael", rev="f879f1942ec1b34b6d3027ce7e4724ad95d15dfa", features = ["xmlsec"] } gcp_auth = "0.9.0" rust_decimal = { version = "^1", features = ["db-postgres", "serde-float"]} -jsonwebtoken = "8.3.0" +# aws_lc_rs, not rust_crypto: the RustCrypto backend refuses RSA keys above 4096 bits, and +# external/guest JWT issuers may sign with 8192-bit keys. +jsonwebtoken = { version = "^10.3", features = ["aws_lc_rs"] } pem = "3.0.1" nix = { version = "0.27.1", features = ["process", "signal"] } fs4 = "0.13" @@ -683,8 +685,8 @@ aws-sdk-bedrockruntime = "=1.122.0" aws-credential-types = "^1" aws-smithy-types = "^1" aws-sdk-secretsmanager = "^1" -aws-sdk-sqs = "=1.77.0" -aws-sdk-sts = "=1.79.0" +aws-sdk-sqs = "^1.89.0" +aws-sdk-sts = "^1.91.0" aws-sdk-sso = "=1.77.0" aws-sdk-ssooidc = "=1.78.0" rustls = "=0.23.35" diff --git a/backend/THREAT_MODEL.md b/backend/THREAT_MODEL.md index af33877078..73a4b53227 100644 --- a/backend/THREAT_MODEL.md +++ b/backend/THREAT_MODEL.md @@ -97,7 +97,7 @@ published advisory history (73 GHSA advisories, several rated 9.9 critical). | id | threat | actor | surface | asset | impact | likelihood | status | controls | evidence | |---|---|---|---|---|---|---|---|---|---| | T1 | SQL injection in app/internal query builders and trigger clauses compromises the metadata DB and connected databases | remote_auth | EP8 | Database, downstream connected systems | critical | almost_certain | partially_mitigated | sqlx parameterized queries elsewhere; query-builder safety reviews | GHSA-225c-j3xq-g6x6, GHSA-78p7-jc72-gv66, GHSA-hvc7-f67h-jx3g, GHSA-wrrg-f89m-f84q, GHSA-79vf-3qwm-2w64, GHSA-55p6-fxj4-v983, GHSA-5g4v-49rj-r52r, GHSA-x6cq-7xr8-53x3, 2cf4bb180b | -| T2 | Server-side request forgery via proxies/executors reaches cloud metadata, internal network, and downstream credentials | remote_auth | EP6, EP7 | Cloud metadata, internal network, downstream connected systems, resource creds | critical | almost_certain | partially_mitigated | SSRF URL validation + redirect-following disabled added piecemeal; MCP private URL access requires the instance-wide `ALLOW_PRIVATE_MCP_SERVER_URLS` opt-in; WebSocket trigger URLs (stored, test, and runnable-resolved) are SSRF-validated at connect time behind the `ALLOW_PRIVATE_WEBSOCKET_URLS` opt-in, and the trigger test route now requires `:write` scope; SAML IdP metadata URLs are SSRF-validated at load time behind the `ALLOW_PRIVATE_SAML_METADATA_URLS` opt-in; outbound network isolation (`clone_newnet`) is opt-in and off by default | GHSA-3ggp-h37f-5qfw, GHSA-98qq-g8rh-xhff, GHSA-hfw8-27mx-63jm, GHSA-3r59-qvvc-774j, GHSA-4pj9-w5jc-g8w7, GHSA-8hh3-jf25-78j5, GHSA-3pjm-4w7f-3r2w, GHSA-f44c-x9hq-h68r, GHSA-j4h4-f8fj-3m3c, 4b06881918, 96a8eb63d4, dbd3942ef3 | +| T2 | Server-side request forgery via proxies/executors reaches cloud metadata, internal network, and downstream credentials | remote_auth | EP6, EP7 | Cloud metadata, internal network, downstream connected systems, resource creds | critical | almost_certain | partially_mitigated | SSRF URL validation + redirect-following disabled added piecemeal; MCP private URL access requires the instance-wide `ALLOW_PRIVATE_MCP_SERVER_URLS` opt-in; WebSocket trigger URLs (stored, test, and runnable-resolved) are SSRF-validated at connect time behind the `ALLOW_PRIVATE_WEBSOCKET_URLS` opt-in, and the trigger test route now requires `:write` scope; SAML IdP metadata URLs are SSRF-validated at load time behind the `ALLOW_PRIVATE_SAML_METADATA_URLS` opt-in; workspace webhook URLs are SSRF-checked when saved, not when sent, behind the `ALLOW_PRIVATE_WEBHOOK_URLS` opt-in; outbound network isolation (`clone_newnet`) is opt-in and off by default | GHSA-3ggp-h37f-5qfw, GHSA-98qq-g8rh-xhff, GHSA-hfw8-27mx-63jm, GHSA-3r59-qvvc-774j, GHSA-4pj9-w5jc-g8w7, GHSA-8hh3-jf25-78j5, GHSA-3pjm-4w7f-3r2w, GHSA-f44c-x9hq-h68r, GHSA-j4h4-f8fj-3m3c, 4b06881918, 96a8eb63d4, dbd3942ef3 | | T3 | Broken authorization / IDOR lets a scoped token or low-privilege member read scripts, job data, and secrets across folders and workspaces | remote_auth | EP5, EP2, EP1 | Scripts, job data, secrets, isolation | critical | almost_certain | partially_mitigated | RLS, token scopes, folder ACLs, view-token HMAC (added incrementally); on managed, sensitive tenants can opt into dedicated DB/worker/namespace, but the shared tier IS the software boundary | GHSA-qfg7-x243-5hg4, GHSA-8x8x-88qc-qp4r, GHSA-2ppx-66jv-wpw5, GHSA-x3x7-g97v-mp59, GHSA-j276-g4h8-g6h5, GHSA-8mv7-hmrg-96xv, GHSA-x2wf-f962-7frq, GHSA-qc7c-gcw6-h4xp, GHSA-vxc5-w28p-m9xw, GHSA-2g34-wfvr-5qqj, GHSA-w7p6-wpxm-pp66, 7edf3f0212, 89a7a37776, ab11c7747a, 664edcdfb7 | | T4 | Remote code execution by injecting attacker-controlled identifiers into generated worker wrappers | remote_auth | EP10 | Worker host, isolation, downstream | critical | likely | partially_mitigated | entrypoint/env-var-name validation added | GHSA-wxjq-w5pj-jqhx, GHSA-5f5q-2vg2-r2x4, GHSA-8q8j-mm3g-5c2q (CVE-2026-33881), bf93657fee, bd05bcadde, 22ec4da5f0 | | T5 | Worker compromise & cross-tenant access via weak-by-default isolation (nsjail off by default → user code runs with only PID-ns `unshare`); sandbox escape where nsjail/dind/podman is enabled | remote_auth | EP9, EP15 | Worker host, isolation, downstream | critical | likely | unmitigated | nsjail off by default everywhere (`DISABLE_NSJAIL=true`); shipped compose gives PID-ns `unshare` only (`FAVOR_UNSHARE_PID=true`), bare installs get no isolation. Where nsjail enabled: read-only remounts, jail-tmp refusal, podman socket gating | GHSA-6qr8-xhg4-453q, GHSA-3vpp-vf62-wqp6, f8467f38c8, df5aec0f5d, f1b6746e0e | diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index 1cb35f927a..c826958128 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -0b72c9e9861cd4636e073e5015e488555196d197 +d0760284ccd3a4443a420839919e0ac60f126650 diff --git a/backend/migrations/20260914133558_draft_base_version.down.sql b/backend/migrations/20260914133558_draft_base_version.down.sql new file mode 100644 index 0000000000..07fc93ba30 --- /dev/null +++ b/backend/migrations/20260914133558_draft_base_version.down.sql @@ -0,0 +1 @@ +ALTER TABLE draft DROP COLUMN base; diff --git a/backend/migrations/20260914133558_draft_base_version.up.sql b/backend/migrations/20260914133558_draft_base_version.up.sql new file mode 100644 index 0000000000..835816d70c --- /dev/null +++ b/backend/migrations/20260914133558_draft_base_version.up.sql @@ -0,0 +1,14 @@ +-- The version a draft forked from, as one opaque text id whatever the kind: a +-- script hash (hex), a flow_version.id, an app_version.id. NULL for a draft that +-- was never forked from a deploy and for kinds that keep no lineage. +ALTER TABLE draft ADD COLUMN base TEXT; + +-- A U+0000 inside a `json` value makes `->>` raise 22P05; such rows keep NULL and +-- get their base on their next save. +UPDATE draft SET base = CASE typ::text + WHEN 'script' THEN value ->> 'parent_hash' + WHEN 'flow' THEN value ->> 'version_id' + ELSE value ->> 'parent_version' + END +WHERE typ::text IN ('script', 'flow', 'app', 'raw_app') + AND position(chr(92) || 'u0000' in replace(value::text, chr(92) || chr(92), '')) = 0; diff --git a/backend/migrations/20260914154128_draft_move.down.sql b/backend/migrations/20260914154128_draft_move.down.sql new file mode 100644 index 0000000000..6fff84e4bb --- /dev/null +++ b/backend/migrations/20260914154128_draft_move.down.sql @@ -0,0 +1 @@ +DROP TABLE draft_move; diff --git a/backend/migrations/20260914154128_draft_move.up.sql b/backend/migrations/20260914154128_draft_move.up.sql new file mode 100644 index 0000000000..e68e36c5ff --- /dev/null +++ b/backend/migrations/20260914154128_draft_move.up.sql @@ -0,0 +1,17 @@ +-- Where an item's drafts went when it moved, so a draft save still addressed to the +-- old path (an editor left open across the move, a chat, the CLI) lands on the moved +-- draft instead of starting a new item there. `email` NULL records a deployed item's +-- move and applies to every user; set, it records that user's draft-only move. +CREATE TABLE draft_move ( + id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + workspace_id VARCHAR(50) NOT NULL REFERENCES workspace(id) ON DELETE CASCADE, + typ DRAFT_KIND NOT NULL, + old_path VARCHAR(255) NOT NULL, + new_path VARCHAR(255) NOT NULL, + email VARCHAR(255) +); + +CREATE INDEX draft_move_old_path_idx ON draft_move (workspace_id, typ, old_path); + +GRANT ALL ON draft_move TO windmill_user; +GRANT ALL ON draft_move TO windmill_admin; diff --git a/backend/migrations/20260919213810_remote_deploy.down.sql b/backend/migrations/20260919213810_remote_deploy.down.sql new file mode 100644 index 0000000000..ae26684ac6 --- /dev/null +++ b/backend/migrations/20260919213810_remote_deploy.down.sql @@ -0,0 +1,4 @@ +DROP TABLE IF EXISTS remote_deploy_token; +ALTER TABLE workspace_settings + DROP COLUMN IF EXISTS remote_deploy_target, + DROP COLUMN IF EXISTS remote_deploy_target_changed_at; diff --git a/backend/migrations/20260919213810_remote_deploy.up.sql b/backend/migrations/20260919213810_remote_deploy.up.sql new file mode 100644 index 0000000000..c32f81fd88 --- /dev/null +++ b/backend/migrations/20260919213810_remote_deploy.up.sql @@ -0,0 +1,35 @@ +-- The workspace on another Windmill instance this workspace deploys into from the UI: +-- `{"base_url": ..., "workspace_id": ...}`, and when it last changed. A token connected under an +-- earlier change counts for nothing, even once the setting points back at the target it was for. +ALTER TABLE workspace_settings + ADD COLUMN remote_deploy_target JSONB, + ADD COLUMN remote_deploy_target_changed_at TIMESTAMPTZ; + +-- One user's token for the remote deploy target, encrypted with the workspace key. +-- `base_url`/`remote_workspace_id` name the target it was granted for: a token is only +-- ever sent to that target, so re-pointing the workspace setting cannot redirect it. +-- Keyed by the account rather than by membership, since a superadmin deploys from workspaces +-- it is not a member of. The account key cascades: whatever deletes or renames an account +-- takes its tokens along, so a later account with the same address cannot inherit them. +CREATE TABLE remote_deploy_token ( + workspace_id VARCHAR(50) NOT NULL REFERENCES workspace(id) ON DELETE CASCADE, + email VARCHAR(255) NOT NULL REFERENCES password(email) ON DELETE CASCADE ON UPDATE CASCADE, + base_url VARCHAR(1000) NOT NULL, + remote_workspace_id VARCHAR(50) NOT NULL, + token TEXT NOT NULL, + remote_email VARCHAR(255) NOT NULL, + -- Part of every proxy URL, and readable only by its owner through the API: a link from + -- elsewhere, which rides the session cookie, cannot know it and so cannot spend the token. + proxy_key VARCHAR(64) NOT NULL, + -- When the connect that wrote the row started, or when a disconnect emptied it. + connected_at TIMESTAMPTZ NOT NULL DEFAULT now(), + -- What the connect ran under: the `created_at` of the owner's membership (NULL for a + -- superadmin with none) and the target's `remote_deploy_target_changed_at`. The row counts + -- only while both are still the same, so a re-add or a target change voids it. + member_since TIMESTAMPTZ, + target_changed_at TIMESTAMPTZ, + PRIMARY KEY (workspace_id, email) +); + +GRANT ALL ON remote_deploy_token TO windmill_user; +GRANT ALL ON remote_deploy_token TO windmill_admin; diff --git a/backend/parsers/windmill-parser-wasm/Cargo.lock b/backend/parsers/windmill-parser-wasm/Cargo.lock index fea3a11063..ccba27bcc5 100644 --- a/backend/parsers/windmill-parser-wasm/Cargo.lock +++ b/backend/parsers/windmill-parser-wasm/Cargo.lock @@ -267,6 +267,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ec6fb3fe69024a75fa7e1bfb48aa6cf59706a101658ea01bfd33b2b248a038f" dependencies = [ "aws-lc-sys", + "untrusted 0.7.1", "zeroize", ] @@ -374,12 +375,6 @@ dependencies = [ "windows-link", ] -[[package]] -name = "base64" -version = "0.13.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" - [[package]] name = "base64" version = "0.21.7" @@ -780,9 +775,9 @@ dependencies = [ [[package]] name = "cmov" -version = "0.5.3" +version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f88a43d011fc4a6876cb7344703e297c71dda42494fee094d5f7c76bf13f746" +checksum = "0c9ea0ac24bc397ab3c98583a3c9ba74fa56b09a4449bbe172b9b1ddb016027a" [[package]] name = "colorchoice" @@ -1337,7 +1332,7 @@ checksum = "da0e4dd2a88388a1f4ccc7c9ce104604dab68d9f408dc34cd45823d5a9069095" dependencies = [ "futures-core", "futures-sink", - "spin 0.9.8", + "spin", ] [[package]] @@ -1535,7 +1530,7 @@ dependencies = [ "cfg-if", "js-sys", "libc", - "wasi", + "wasi 0.11.1+wasi-snapshot-preview1", "wasm-bindgen", ] @@ -2186,16 +2181,20 @@ dependencies = [ [[package]] name = "jsonwebtoken" -version = "8.3.0" +version = "10.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6971da4d9c3aa03c3d8f3ff0f4155b534aad021292003895a469716b2a230378" +checksum = "eba32bfb4ffdeaca3e34431072faf01745c9b26d25504aa7a6cf5684334fc4fc" dependencies = [ - "base64 0.21.7", + "aws-lc-rs", + "base64 0.22.1", + "getrandom 0.2.17", + "js-sys", "pem", - "ring 0.16.20", "serde", "serde_json", + "signature", "simple_asn1", + "zeroize", ] [[package]] @@ -2225,7 +2224,7 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" dependencies = [ - "spin 0.9.8", + "spin", ] [[package]] @@ -2581,7 +2580,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a69bcab0ad47271a0234d9422b131806bf3968021e5dc9328caf2d4cd58557fc" dependencies = [ "libc", - "wasi", + "wasi 0.11.1+wasi-snapshot-preview1", "windows-sys 0.61.2", ] @@ -2598,7 +2597,7 @@ dependencies = [ "httparse", "memchr", "mime", - "spin 0.9.8", + "spin", "version_check", ] @@ -2817,7 +2816,7 @@ dependencies = [ "num-integer", "num-iter", "num-traits", - "rand 0.8.5", + "rand 0.8.6", "smallvec", "zeroize", ] @@ -2903,15 +2902,14 @@ checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381" [[package]] name = "openssl" -version = "0.10.78" +version = "0.10.80" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f38c4372413cdaaf3cc79dd92d29d7d9f5ab09b51b10dded508fb90bb70b9222" +checksum = "a45fa2aa886c42762255da344f0a0d313e254066c46aad76f300c3d3da62d967" dependencies = [ "bitflags", "cfg-if", "foreign-types", "libc", - "once_cell", "openssl-macros", "openssl-sys", ] @@ -2935,9 +2933,9 @@ checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" [[package]] name = "openssl-sys" -version = "0.9.114" +version = "0.9.117" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13ce1245cd07fcc4cfdb438f7507b0c7e4f3849a69fd84d52374c66d83741bb6" +checksum = "b47e7e6bb2c38cd930d25a23b40fa52e068c10e85f3e03a7f5ba5aaca5713695" dependencies = [ "cc", "libc", @@ -2994,11 +2992,12 @@ checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" [[package]] name = "pem" -version = "1.1.1" +version = "3.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8835c273a76a90455d7344889b0964598e3316e2a79ede8e36f16bdcf2228b8" +checksum = "1d30c53c26bc5b31a98cd02d20f25a7c8567146caf63ed593a9d87b2775291be" dependencies = [ - "base64 0.13.1", + "base64 0.22.1", + "serde_core", ] [[package]] @@ -3047,6 +3046,16 @@ dependencies = [ "phf_shared 0.12.1", ] +[[package]] +name = "phf" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1562dc717473dbaa4c1f85a36410e03c047b2e7df7f45ee938fbef64ae7fadf" +dependencies = [ + "phf_shared 0.13.1", + "serde", +] + [[package]] name = "phf_codegen" version = "0.11.3" @@ -3064,7 +3073,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3c80231409c20246a13fddb31776fb942c38553c51e871f8cbd687a4cfb5843d" dependencies = [ "phf_shared 0.11.3", - "rand 0.8.5", + "rand 0.8.6", ] [[package]] @@ -3098,6 +3107,15 @@ dependencies = [ "siphasher 1.0.2", ] +[[package]] +name = "phf_shared" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e57fef6bc5981e38c2ce2d63bfa546861309f875b8a75f092d1d54ae2d64f266" +dependencies = [ + "siphasher 1.0.2", +] + [[package]] name = "php-parser-rs" version = "0.1.3" @@ -3169,9 +3187,9 @@ dependencies = [ [[package]] name = "postgres-protocol" -version = "0.6.11" +version = "0.6.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56201207dac53e2f38e848e31b4b91616a6bb6e0c7205b77718994a7f49e70fc" +checksum = "08808e3c483c46e999108051c78334f473d5adb59d78bb80a1268c7e6aa6c514" dependencies = [ "base64 0.22.1", "byteorder", @@ -3187,9 +3205,9 @@ dependencies = [ [[package]] name = "postgres-types" -version = "0.2.9" +version = "0.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "613283563cd90e1dfc3518d548caee47e0e725455ed619881f5cf21f36de4b48" +checksum = "851ca9db4932932d69f3ea811b1abe63087a0f740a47692619dd40d4899b68be" dependencies = [ "array-init", "bit-vec 0.6.3", @@ -3197,7 +3215,7 @@ dependencies = [ "chrono", "fallible-iterator", "postgres-protocol", - "serde", + "serde_core", "serde_json", "uuid", ] @@ -3394,16 +3412,16 @@ dependencies = [ [[package]] name = "quinn-proto" -version = "0.11.14" +version = "0.11.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "434b42fec591c96ef50e21e886936e66d3cc3f737104fdb9b737c40ffb94c098" +checksum = "4fcb935c5bec503c2f0e306bdd3e58bb9029dcb14fa8d9ac76e3a5256ac0763e" dependencies = [ "aws-lc-rs", "bytes", "getrandom 0.3.4", "lru-slab", - "rand 0.9.0", - "ring 0.17.14", + "rand 0.9.5", + "ring", "rustc-hash 2.1.1", "rustls", "rustls-pki-types", @@ -3457,9 +3475,9 @@ checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09" [[package]] name = "rand" -version = "0.8.5" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +checksum = "5ca0ecfa931c29007047d1bc58e623ab12e5590e8c7cc53200d5202b69266d8a" dependencies = [ "libc", "rand_chacha 0.3.1", @@ -3468,13 +3486,12 @@ dependencies = [ [[package]] name = "rand" -version = "0.9.0" +version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3779b94aeb87e8bd4e834cee3650289ee9e0d5677f976ecdb6d219e5f4f6cd94" +checksum = "b9ef1d0d795eb7d84685bca4f72f3649f064e6641543d3a8c415898726a57b41" dependencies = [ "rand_chacha 0.9.0", "rand_core 0.9.5", - "zerocopy", ] [[package]] @@ -3752,22 +3769,7 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "503c78f59814e2664c9980b739b19e40f549233ecf39928040ee54fdd431f614" dependencies = [ - "rand 0.8.5", -] - -[[package]] -name = "ring" -version = "0.16.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3053cf52e236a3ed746dfc745aa9cacf1b791d846bdaf412f60a8d7d6e17c8fc" -dependencies = [ - "cc", - "libc", - "once_cell", - "spin 0.5.2", - "untrusted 0.7.1", - "web-sys", - "winapi", + "rand 0.8.6", ] [[package]] @@ -3843,7 +3845,7 @@ dependencies = [ "borsh", "bytes", "num-traits", - "rand 0.8.5", + "rand 0.8.6", "rkyv", "serde", "serde_json", @@ -3901,7 +3903,7 @@ checksum = "758025cb5fccfd3bc2fd74708fd4682be41d99e5dff73c377c0646c6012c73a4" dependencies = [ "aws-lc-rs", "once_cell", - "ring 0.17.14", + "ring", "rustls-pki-types", "rustls-webpki", "subtle", @@ -3959,12 +3961,12 @@ checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f" [[package]] name = "rustls-webpki" -version = "0.103.9" +version = "0.103.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7df23109aa6c1567d1c575b9952556388da57401e4ace1d15f79eedad0d8f53" +checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e" dependencies = [ "aws-lc-rs", - "ring 0.17.14", + "ring", "rustls-pki-types", "untrusted 0.9.0", ] @@ -4199,15 +4201,16 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.143" +version = "1.0.151" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d401abef1d108fbd9cbaebc3e46611f4b1021f714a0597a71f41ee463f5f4a5a" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" dependencies = [ "indexmap", "itoa", "memchr", - "ryu", "serde", + "serde_core", + "zmij", ] [[package]] @@ -4426,12 +4429,6 @@ dependencies = [ "windows-sys 0.61.2", ] -[[package]] -name = "spin" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" - [[package]] name = "spin" version = "0.9.8" @@ -4594,7 +4591,7 @@ dependencies = [ "memchr", "once_cell", "percent-encoding", - "rand 0.8.5", + "rand 0.8.6", "rsa", "serde", "sha1", @@ -4605,7 +4602,7 @@ dependencies = [ "thiserror 2.0.18", "tracing", "uuid", - "whoami", + "whoami 1.6.1", ] [[package]] @@ -4636,7 +4633,7 @@ dependencies = [ "memchr", "num-bigint", "once_cell", - "rand 0.8.5", + "rand 0.8.6", "serde", "serde_json", "sha2 0.10.9", @@ -4646,7 +4643,7 @@ dependencies = [ "thiserror 2.0.18", "tracing", "uuid", - "whoami", + "whoami 1.6.1", ] [[package]] @@ -5063,9 +5060,9 @@ checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" [[package]] name = "tar" -version = "0.4.45" +version = "0.4.46" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22692a6476a21fa75fdfc11d452fda482af402c008cdbaf3476414e122040973" +checksum = "3f6221d9a6003c78398e3b239969f352578258df48c8eb051caadae0015bc840" dependencies = [ "filetime", "libc", @@ -5274,9 +5271,9 @@ dependencies = [ [[package]] name = "tokio-postgres" -version = "0.7.13" +version = "0.7.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c95d533c83082bb6490e0189acaa0bbeef9084e60471b696ca6988cd0541fb0" +checksum = "a528f7d280f6d5b9cd149635c8705b0dd049754bc67d81d31fa25169a93809d3" dependencies = [ "async-trait", "byteorder", @@ -5287,15 +5284,15 @@ dependencies = [ "log", "parking_lot", "percent-encoding", - "phf 0.11.3", + "phf 0.13.1", "pin-project-lite", "postgres-protocol", "postgres-types", - "rand 0.9.0", - "socket2 0.5.10", + "rand 0.10.1", + "socket2 0.6.3", "tokio", "tokio-util", - "whoami", + "whoami 2.1.0", ] [[package]] @@ -5797,7 +5794,7 @@ dependencies = [ "getopts", "log", "phf_codegen", - "rand 0.8.5", + "rand 0.8.6", ] [[package]] @@ -5919,6 +5916,15 @@ version = "0.11.1+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" +[[package]] +name = "wasi" +version = "0.14.7+wasi-0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "883478de20367e224c0090af9cf5f9fa85bed63a95c1abf3afc5c083ebc06e8c" +dependencies = [ + "wasip2", +] + [[package]] name = "wasip2" version = "1.0.2+wasi-0.2.9" @@ -5943,6 +5949,15 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b8dad83b4f25e74f184f64c43b150b91efe7647395b42289f38e50566d82855b" +[[package]] +name = "wasite" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66fe902b4a6b8028a753d5424909b764ccf79b7a209eac9bf97e59cda9f71a42" +dependencies = [ + "wasi 0.14.7+wasi-0.2.4", +] + [[package]] name = "wasm-bindgen" version = "0.2.103" @@ -6154,7 +6169,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5d4a4db5077702ca3015d3d02d74974948aba2ad9e12ab7df718ee64ccd7e97d" dependencies = [ "libredox", - "wasite", + "wasite 0.1.0", +] + +[[package]] +name = "whoami" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fae98cf96deed1b7572272dfc777713c249ae40aa1cf8862e091e8b745f5361" +dependencies = [ + "libredox", + "wasite 1.0.2", "web-sys", ] @@ -6191,7 +6216,7 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "windmill-common" -version = "1.815.0" +version = "1.817.0" dependencies = [ "aho-corasick", "anyhow", @@ -6233,9 +6258,10 @@ dependencies = [ "pep440_rs", "phf 0.11.3", "pin-project-lite", + "pkcs1", "postgres-native-tls", "quick_cache", - "rand 0.9.0", + "rand 0.9.5", "regex", "reqwest", "reqwest-middleware", @@ -6244,8 +6270,10 @@ dependencies = [ "serde", "serde_json", "serde_yml", + "sha1", "sha2 0.10.9", "size", + "spki", "sqlx", "strum 0.27.2", "strum_macros 0.27.2", @@ -6274,7 +6302,7 @@ dependencies = [ [[package]] name = "windmill-macros" -version = "1.815.0" +version = "1.817.0" dependencies = [ "proc-macro2", "quote", @@ -6286,7 +6314,7 @@ dependencies = [ [[package]] name = "windmill-parser" -version = "1.815.0" +version = "1.817.0" dependencies = [ "convert_case", "serde", @@ -6295,7 +6323,7 @@ dependencies = [ [[package]] name = "windmill-parser-bash" -version = "1.815.0" +version = "1.817.0" dependencies = [ "anyhow", "lazy_static", @@ -6307,7 +6335,7 @@ dependencies = [ [[package]] name = "windmill-parser-csharp" -version = "1.815.0" +version = "1.817.0" dependencies = [ "anyhow", "serde_json", @@ -6319,7 +6347,7 @@ dependencies = [ [[package]] name = "windmill-parser-go" -version = "1.815.0" +version = "1.817.0" dependencies = [ "anyhow", "gosyn", @@ -6331,7 +6359,7 @@ dependencies = [ [[package]] name = "windmill-parser-graphql" -version = "1.815.0" +version = "1.817.0" dependencies = [ "anyhow", "lazy_static", @@ -6343,7 +6371,7 @@ dependencies = [ [[package]] name = "windmill-parser-java" -version = "1.815.0" +version = "1.817.0" dependencies = [ "anyhow", "serde_json", @@ -6355,7 +6383,7 @@ dependencies = [ [[package]] name = "windmill-parser-nu" -version = "1.815.0" +version = "1.817.0" dependencies = [ "anyhow", "nu-parser", @@ -6366,7 +6394,7 @@ dependencies = [ [[package]] name = "windmill-parser-php" -version = "1.815.0" +version = "1.817.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -6377,7 +6405,7 @@ dependencies = [ [[package]] name = "windmill-parser-py" -version = "1.815.0" +version = "1.817.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -6389,7 +6417,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-asset" -version = "1.815.0" +version = "1.817.0" dependencies = [ "anyhow", "rustpython-ast", @@ -6400,7 +6428,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-imports" -version = "1.815.0" +version = "1.817.0" dependencies = [ "anyhow", "async-recursion", @@ -6422,7 +6450,7 @@ dependencies = [ [[package]] name = "windmill-parser-r" -version = "1.815.0" +version = "1.817.0" dependencies = [ "anyhow", "serde_json", @@ -6434,7 +6462,7 @@ dependencies = [ [[package]] name = "windmill-parser-ruby" -version = "1.815.0" +version = "1.817.0" dependencies = [ "anyhow", "lazy_static", @@ -6448,7 +6476,7 @@ dependencies = [ [[package]] name = "windmill-parser-rust" -version = "1.815.0" +version = "1.817.0" dependencies = [ "anyhow", "convert_case", @@ -6465,7 +6493,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql" -version = "1.815.0" +version = "1.817.0" dependencies = [ "anyhow", "lazy_static", @@ -6478,7 +6506,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql-asset" -version = "1.815.0" +version = "1.817.0" dependencies = [ "anyhow", "serde", @@ -6490,7 +6518,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts" -version = "1.815.0" +version = "1.817.0" dependencies = [ "anyhow", "lazy_static", @@ -6508,7 +6536,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts-asset" -version = "1.815.0" +version = "1.817.0" dependencies = [ "anyhow", "serde-wasm-bindgen", @@ -6524,7 +6552,7 @@ dependencies = [ [[package]] name = "windmill-parser-wac" -version = "1.815.0" +version = "1.817.0" dependencies = [ "anyhow", "rustpython-ast", @@ -6540,7 +6568,7 @@ dependencies = [ [[package]] name = "windmill-parser-wasm" -version = "1.815.0" +version = "1.817.0" dependencies = [ "anyhow", "getrandom 0.2.17", @@ -6572,7 +6600,7 @@ dependencies = [ [[package]] name = "windmill-parser-yaml" -version = "1.815.0" +version = "1.817.0" dependencies = [ "anyhow", "lazy_static", @@ -6586,14 +6614,14 @@ dependencies = [ [[package]] name = "windmill-types" -version = "1.815.0" +version = "1.817.0" dependencies = [ "anyhow", "bitflags", "chrono", "hex", "itertools 0.14.0", - "rand 0.9.0", + "rand 0.9.5", "serde", "serde_json", "sqlx", @@ -7282,6 +7310,20 @@ name = "zeroize" version = "1.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b97154e67e32c85465826e8bcc1c59429aaaf107c1e4a9e53c8d8ccd5eff88d0" +dependencies = [ + "zeroize_derive", +] + +[[package]] +name = "zeroize_derive" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c50655cbb0fe3fc43170059e702f1ce5e19b84cec58dc87b037a09935c2f328" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] [[package]] name = "zerotrie" @@ -7315,3 +7357,9 @@ dependencies = [ "quote", "syn 2.0.117", ] + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/backend/parsers/windmill-parser-wasm/Cargo.toml b/backend/parsers/windmill-parser-wasm/Cargo.toml index 2f700b7b01..0244b7b106 100644 --- a/backend/parsers/windmill-parser-wasm/Cargo.toml +++ b/backend/parsers/windmill-parser-wasm/Cargo.toml @@ -12,7 +12,7 @@ resolver = "2" members = ["."] [workspace.package] -version = "1.815.0" +version = "1.817.0" edition = "2021" authors = ["Ruben Fiszel "] diff --git a/backend/src/monitor.rs b/backend/src/monitor.rs index 750bc40560..7e4dbb820a 100644 --- a/backend/src/monitor.rs +++ b/backend/src/monitor.rs @@ -1953,20 +1953,6 @@ pub async fn delete_expired_items(db: &DB) -> () { tracing::error!("Error deleting expired shared AI artifacts: {:?}", e); } - let audit_retention_days = audit_log_retention_days().await; - let audit_retention_secs: i64 = audit_retention_days * 60 * 60 * 24; - - // Clean up old (non-partitioned) audit table — will eventually be empty and dropped - if let Err(e) = sqlx::query_scalar!( - "DELETE FROM audit WHERE timestamp <= now() - ($1::bigint::text || ' s')::interval", - audit_retention_secs, - ) - .fetch_all(db) - .await - { - tracing::error!("Error deleting audit log: {:?}", e); - } - if let Err(e) = sqlx::query_scalar!( "DELETE FROM autoscaling_event WHERE applied_at <= now() - ($1::bigint::text || ' s')::interval", 30 * 24 * 60 * 60, // 30 days diff --git a/backend/summarized_schema.txt b/backend/summarized_schema.txt index 47f9faca68..169eb81681 100644 --- a/backend/summarized_schema.txt +++ b/backend/summarized_schema.txt @@ -56,7 +56,7 @@ app_version_lite: id(bigint), value(jsonb) FK: (id) -> app_version(id) asset: workspace_id(char), path(char), kind(asset_kind), usage_access_type(asset_access_type), usage_path(char), usage_kind(asset_usage_kind), created_at(ts), id(bigint), columns(jsonb) FK: (workspace_id) -> workspace(id) -audit: workspace_id(char), id(bigint), timestamp(ts), username(char), operation(char), action_kind(action_kind), resource(char), parameters(jsonb), email(char), span(char) +audit_partitioned: workspace_id(char), id(bigint), timestamp(ts), username(char), operation(char), action_kind(action_kind), resource(char), parameters(jsonb), email(char), span(char) autoscaling_event: id(int), worker_group(text), event_type(autoscaling_event_type), desired_workers(int), applied_at(ts), reason(text) capture: workspace_id(char), path(char), created_at(ts), created_by(char), main_args(jsonb), is_flow(bool), trigger_kind(trigger_kind), preprocessor_args(jsonb), id(bigint) FK: (workspace_id) -> workspace(id) @@ -171,6 +171,9 @@ postgres_trigger: path(char), script_path(char), is_flow(bool), workspace_id(cha FK: (workspace_id) -> workspace(id) raw_app: path(char), version(int), workspace_id(char), summary(char), edited_at(ts), data(text), extra_perms(jsonb), labels(text[]) FK: (workspace_id) -> workspace(id) +remote_deploy_token: workspace_id(char), email(char), base_url(char), remote_workspace_id(char), token(text), remote_email(char), proxy_key(char), connected_at(timestamptz), member_since(timestamptz), target_changed_at(timestamptz) + FK: (email) -> password(email) + FK: (workspace_id) -> workspace(id) resource: workspace_id(char), path(char), value(jsonb), description(text), resource_type(char), extra_perms(jsonb), edited_at(ts), created_by(char), labels(text[]) FK: (workspace_id) -> workspace(id) resource_type: workspace_id(char), name(char), schema(jsonb), description(text), edited_at(ts), created_by(char), format_extension(char), is_fileset(bool), display_name(char) @@ -234,7 +237,7 @@ workspace_protection_rule: workspace_id(char), name(char), rules(int), bypass_gr FK: (workspace_id) -> workspace(id) workspace_runnable_dependencies: flow_path(char), runnable_path(char), script_hash(bigint), runnable_is_flow(bool), workspace_id(char), app_path(char), id(bigint), runnable_is_agent(bool) FK: (app_path, workspace_id) -> app(path, workspace_id) | (flow_path, workspace_id) -> flow(path, workspace_id) -workspace_settings: workspace_id(char), slack_team_id(char), slack_name(char), slack_command_script(char), slack_email(char), customer_id(char), plan(char), webhook(text), ai_config(jsonb), large_file_storage(jsonb), git_sync(jsonb), default_app(char), default_scripts(jsonb), deploy_ui(jsonb), mute_critical_alerts(bool), color(char), operator_settings(jsonb), teams_command_script(text), teams_team_id(text), teams_team_name(text), git_app_installations(jsonb), ducklake(jsonb), slack_oauth_client_id(char), slack_oauth_client_secret(char), datatable(jsonb), teams_team_guid(text), auto_invite(jsonb), error_handler(jsonb), success_handler(jsonb), public_app_execution_limit_per_minute(int), dbt_warehouses(jsonb), guest_access_enabled(bool), guest_jwt_public_key(text), guest_jwt_jwks_url(text), ai_sessions_backup_generation(int), add_admins_and_developers_to_forks(bool) +workspace_settings: workspace_id(char), slack_team_id(char), slack_name(char), slack_command_script(char), slack_email(char), customer_id(char), plan(char), webhook(text), ai_config(jsonb), large_file_storage(jsonb), git_sync(jsonb), default_app(char), default_scripts(jsonb), deploy_ui(jsonb), mute_critical_alerts(bool), color(char), operator_settings(jsonb), teams_command_script(text), teams_team_id(text), teams_team_name(text), git_app_installations(jsonb), ducklake(jsonb), slack_oauth_client_id(char), slack_oauth_client_secret(char), datatable(jsonb), teams_team_guid(text), auto_invite(jsonb), error_handler(jsonb), success_handler(jsonb), public_app_execution_limit_per_minute(int), dbt_warehouses(jsonb), guest_access_enabled(bool), guest_jwt_public_key(text), guest_jwt_jwks_url(text), ai_sessions_backup_generation(int), add_admins_and_developers_to_forks(bool), remote_deploy_target(jsonb), remote_deploy_target_changed_at(ts) FK: (workspace_id) -> workspace(id) zombie_job_counter: job_id(uuid), counter(int) FK: (job_id) -> v2_job(id) diff --git a/backend/tests/app_version_order.rs b/backend/tests/app_version_order.rs new file mode 100644 index 0000000000..e3557ad55c --- /dev/null +++ b/backend/tests/app_version_order.rs @@ -0,0 +1,193 @@ +//! The deployed order of an app's versions is the order they were appended to +//! `app.versions`, not the order of their `created_at`. +//! +//! `app_version.created_at` defaults to `now()`, which in Postgres is the +//! transaction's start time, while the append happens under the app row's lock. +//! Two deploys that overlap therefore land in one order and carry timestamps in +//! the other. The head the editor guards against, and the sequence the diff +//! picker numbers, both have to follow the array. +//! +//! Users from the `base` fixture: test-user (admin, token SECRET_TOKEN). + +use serde_json::json; +use sqlx::{Pool, Postgres}; +use windmill_test_utils::*; + +const TOKEN: &str = "SECRET_TOKEN"; + +#[sqlx::test(fixtures("base"))] +async fn test_app_head_follows_the_append_order_not_the_timestamps( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let ws = format!( + "http://localhost:{}/api/w/test-workspace", + server.addr.port() + ); + let client = reqwest::Client::new(); + + let res = client + .post(format!("{ws}/apps/create")) + .header("Authorization", format!("Bearer {TOKEN}")) + .json(&json!({ + "path": "u/test-user/order_app", + "summary": "ordered", + "value": {}, + "policy": { "execution_mode": "publisher", "triggerables": {} } + })) + .send() + .await?; + assert!(res.status().is_success(), "{}", res.text().await?); + + let first: i64 = sqlx::query_scalar( + "SELECT versions[array_upper(versions, 1)] FROM app + WHERE workspace_id = 'test-workspace' AND path = 'u/test-user/order_app'", + ) + .fetch_one(&db) + .await?; + + // The overlapping deploy: appended after `first`, so it is the version that + // landed, but stamped before it, so a timestamp sort puts it underneath. + let second: i64 = sqlx::query_scalar( + "INSERT INTO app_version (app_id, value, created_by, created_at, raw_app) + SELECT app_id, value, 'racer', created_at - interval '1 hour', raw_app + FROM app_version WHERE id = $1 + RETURNING id", + ) + .bind(first) + .fetch_one(&db) + .await?; + sqlx::query( + "UPDATE app SET versions = array_append(versions, $1::bigint) + WHERE workspace_id = 'test-workspace' AND path = 'u/test-user/order_app'", + ) + .bind(second) + .execute(&db) + .await?; + + let head: serde_json::Value = client + .get(format!( + "{ws}/apps/get_latest_version/u/test-user/order_app" + )) + .header("Authorization", format!("Bearer {TOKEN}")) + .send() + .await? + .json() + .await?; + assert_eq!( + head["version"], second, + "the head is the version appended last, not the newest timestamp: {head}" + ); + + let history: Vec = client + .get(format!("{ws}/apps/history/p/u/test-user/order_app")) + .header("Authorization", format!("Bearer {TOKEN}")) + .send() + .await? + .json() + .await?; + let listed: Vec = history + .iter() + .map(|v| v["version"].as_i64().unwrap()) + .collect(); + assert_eq!( + listed, + vec![second, first], + "the history lists in deployed order, so it leads with the head" + ); + + // Enough versions that a page cannot hold them, so "asked for nothing" and "asked for + // a page" are visibly different answers. + let mut appended = vec![second, first]; + for _ in 0..24 { + let extra: i64 = sqlx::query_scalar( + "INSERT INTO app_version (app_id, value, created_by, created_at, raw_app) + SELECT app_id, value, 'bulk', created_at, raw_app FROM app_version WHERE id = $1 + RETURNING id", + ) + .bind(first) + .fetch_one(&db) + .await?; + sqlx::query( + "UPDATE app SET versions = array_append(versions, $1::bigint) + WHERE workspace_id = 'test-workspace' AND path = 'u/test-user/order_app'", + ) + .bind(extra) + .execute(&db) + .await?; + appended.insert(0, extra); + } + + let versions_at = |query: &str| { + let url = format!("{ws}/apps/history/p/u/test-user/order_app{query}"); + let client = client.clone(); + async move { + let rows: Vec = client + .get(url) + .header("Authorization", format!("Bearer {TOKEN}")) + .send() + .await? + .json() + .await?; + Ok::<_, anyhow::Error>( + rows.iter() + .map(|v| v["version"].as_i64().unwrap()) + .collect::>(), + ) + } + }; + + // The deployment-history panel and the CLI read this endpoint without paging, so + // asking for no page has to keep answering with the whole history. + assert_eq!( + versions_at("").await?, + appended, + "an unpaginated request still answers whole" + ); + assert_eq!( + versions_at("?per_page=10").await?, + appended[..10], + "a page holds what was asked for, newest first" + ); + assert_eq!( + versions_at("?per_page=10&page=2").await?, + appended[10..20], + "the next page carries on where the first left off, skipping nothing" + ); + // A page past the end runs off it rather than overflowing into one. (The clamp on an + // asked-for size is pinned where it lives, in `paginate_optional`'s own test.) + assert!( + versions_at("?per_page=10&page=99999999") + .await? + .is_empty(), + "a page past the end is empty" + ); + // Naming only the page still asks for a page, rather than an offset into a listing + // sized to hold everything — which would answer nothing at all. + assert_eq!( + versions_at("?page=1").await?, + appended, + "a first page with no size named holds the history" + ); + + // A version that never entered the deployed sequence — one a restore or a fork copy + // could leave behind — still has to be reachable, after the ones that did. + let stray: i64 = sqlx::query_scalar( + "INSERT INTO app_version (app_id, value, created_by, created_at, raw_app) + SELECT app_id, value, 'stray', created_at, raw_app FROM app_version WHERE id = $1 + RETURNING id", + ) + .bind(first) + .fetch_one(&db) + .await?; + let mut with_stray = appended.clone(); + with_stray.push(stray); + assert_eq!( + versions_at("").await?, + with_stray, + "a version outside app.versions is listed, and after the deployed sequence" + ); + + Ok(()) +} diff --git a/backend/tests/bun_jobs.rs b/backend/tests/bun_jobs.rs index 63d5341ba3..14f8c422a4 100644 --- a/backend/tests/bun_jobs.rs +++ b/backend/tests/bun_jobs.rs @@ -1303,6 +1303,62 @@ async fn test_bun_wac_inline_task_cache_is_per_task(db: Pool) -> anyho Ok(()) } +/// A task that picks its own worker tag is held to CUSTOM_TAGS like a flow step: with none +/// set, only a superadmin may send it to `gpu`, so the workflow fails instead of queueing it. +#[sqlx::test(fixtures("base"))] +async fn test_bun_wac_task_tag_outside_custom_tags(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + let content = r#" +import { workflow, task } from "windmill-client"; + +const onGpu = task(async (n: number) => { + return n * 2; +}, { tag: "gpu" }); + +export const main = workflow(async (n: number) => { + return await onGpu(n); +}); +"# + .to_owned(); + + let job = RunJob::from(JobPayload::Code(RawCode { + hash: None, + content, + path: None, + language: ScriptLang::Bun, + lock: None, + concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default() + .into(), + debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), + cache_ttl: None, + cache_ignore_s3_path: None, + dedicated_worker: None, + modules: None, + tag: None, + })) + .arg("n", serde_json::json!(5)) + .as_user("test-user-2", "test2@windmill.dev") + .run_until_complete(&db, false, port) + .await; + + assert!(!job.success, "the workflow must fail"); + let result = job.json_result().unwrap(); + let message = result["error"]["message"].as_str().unwrap_or_default(); + assert!( + message.contains("task 'onGpu' cannot run on tag 'gpu'"), + "unexpected error: {result}" + ); + let children: i64 = sqlx::query_scalar("SELECT count(*) FROM v2_job WHERE parent_job = $1") + .bind(job.id) + .fetch_one(&db) + .await?; + assert_eq!(children, 0, "no child is queued on the refused tag"); + Ok(()) +} + // ============================================================================ // Environment Variable Tests // ============================================================================ diff --git a/backend/tests/datatable_migration_add_down.rs b/backend/tests/datatable_migration_add_down.rs index 3cd39e0fa3..47ea4c253b 100644 --- a/backend/tests/datatable_migration_add_down.rs +++ b/backend/tests/datatable_migration_add_down.rs @@ -142,3 +142,35 @@ async fn test_add_down_to_an_applied_migration(db: Pool) -> anyhow::Re Ok(()) } + +/// A definition saved before `-- role` annotations were parsed can open with a comment that now +/// reads as a malformed one. It never gets through a role check again, so an admin must still be +/// able to finish it rather than be refused on its annotation. +#[sqlx::test(fixtures("datatable_migrations_grants"))] +async fn test_add_down_to_a_migration_whose_comment_predates_role_annotations( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + setup_applied_migration_without_down(&db).await?; + let code_up = format!("-- Role table for widgets\n{CODE_UP}"); + sqlx::query("UPDATE datatable_migrations SET code_up = $1 WHERE workspace_id = 'dtmig-ws'") + .bind(&code_up) + .execute(&db) + .await?; + + let server = ApiServer::start(db.clone()).await?; + let resp = authed(reqwest::Client::new().post(format!( + "http://localhost:{}/api/w/dtmig-ws/workspaces/upsert_datatable_migration/main", + server.addr.port() + ))) + .json(&json!({ + "timestamp": VERSION, + "name": "create_widgets", + "code_up": code_up, + "code_down": CODE_DOWN, + })) + .send() + .await?; + assert_eq!(resp.status(), 200, "{}", resp.text().await?); + Ok(()) +} diff --git a/backend/tests/dependency_map.rs b/backend/tests/dependency_map.rs index 4520cae848..f4b74525fb 100644 --- a/backend/tests/dependency_map.rs +++ b/backend/tests/dependency_map.rs @@ -521,10 +521,8 @@ def main(): .await .unwrap(); - assert_eq!( - res.text().await.unwrap(), - "app f/rel/root_app updated (npath: \"f/rel/root_app_renamed\")" - ); + let deployed: serde_json::Value = res.json().await.unwrap(); + assert_eq!(deployed["path"], "f/rel/root_app_renamed", "{deployed}"); let mut completed = listen_for_completed_jobs(&db).await; in_test_worker(&db, completed.next(), port).await; diff --git a/backend/tests/drafts_move_mirror.rs b/backend/tests/drafts_move_mirror.rs new file mode 100644 index 0000000000..d619381c46 --- /dev/null +++ b/backend/tests/drafts_move_mirror.rs @@ -0,0 +1,74 @@ +//! Moving a draft must carry both of its path keys. +//! +//! A draft value holds a typed path and a mirror the editors keep beside it while +//! it differs from the row's path (`path`/`draft_path`; which is which depends on +//! the kind). The loaders prefer the mirror, so a move that rewrote only the typed +//! key left the mirror naming the old location: reopening the item restored the +//! old path, and the next autosave wrote it back — undoing the move silently. +//! +//! This pins `move_draft`, including that a draft with no mirror never gains one. +//! A deploy's carry follows the keys only where they still name the old path +//! (`drafts_save_follows_move` pins that). + +use serde_json::Value; +use sqlx::{Pool, Postgres}; + +use windmill_test_utils::*; + +async fn move_to(port: u16, from: &str, to: &str) -> anyhow::Result { + let resp = reqwest::Client::new() + .post(format!( + "http://localhost:{port}/api/w/test-workspace/drafts/move/script/{from}" + )) + .header("Authorization", "Bearer SECRET_TOKEN") + .json(&serde_json::json!({ "new_path": to })) + .send() + .await?; + // Asserted here so a rejected move fails as itself, rather than as a JSON + // parse error in the read-back below. + let status = resp.status(); + let body = resp.text().await?; + assert!(status.is_success(), "move {from} -> {to} failed: {body}"); + Ok(body) +} + +/// The stored draft value at `path`, read back through the API so this test needs +/// no `sqlx::query!` (which would want an offline cache entry of its own). +async fn value_at(port: u16, path: &str) -> anyhow::Result { + let body: Value = reqwest::Client::new() + .get(format!( + "http://localhost:{port}/api/w/test-workspace/drafts/get_own/script/{path}" + )) + .header("Authorization", "Bearer SECRET_TOKEN") + .send() + .await? + .json() + .await?; + Ok(body + .get("value") + .cloned() + .unwrap_or_else(|| panic!("no draft at {path}: {body}"))) +} + +#[sqlx::test(fixtures("base", "drafts_move_mirror"))] +async fn test_move_carries_both_path_keys(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + move_to(port, "u/test-user/draft_mirror", "u/test-user/renamed").await?; + move_to(port, "u/test-user/draft_plain", "u/test-user/plain2").await?; + + // The mirror follows: left at `u/test-user/friendly` it would win at load and + // walk the item back there. + let moved = value_at(port, "u/test-user/renamed").await?; + assert_eq!(moved["path"], "u/test-user/renamed"); + assert_eq!(moved["draft_path"], "u/test-user/renamed"); + + // A draft that never had a mirror must not be given one. + let plain = value_at(port, "u/test-user/plain2").await?; + assert_eq!(plain["path"], "u/test-user/plain2"); + assert_eq!(plain.get("draft_path"), None, "mirror injected: {plain}"); + + Ok(()) +} diff --git a/backend/tests/drafts_move_taken.rs b/backend/tests/drafts_move_taken.rs new file mode 100644 index 0000000000..40318ca2ef --- /dev/null +++ b/backend/tests/drafts_move_taken.rs @@ -0,0 +1,250 @@ +//! A rename onto a path that already holds a draft is refused. +//! +//! Nothing deployed can sit at a rename's destination (the deploy conflicts on +//! that), but a draft can: a never-deployed item, or a draft left on an archived +//! script. Moving onto it would merge two items or strand a row, so the rename +//! itself fails, in its own transaction, and the source stays deployed. The +//! destination draft here is the deployer's own, which is the same collision. + +use serde_json::{json, Value}; +use sqlx::{Pool, Postgres}; + +use windmill_test_utils::*; + +/// Hex form of script hash 7010, the way the API takes a parent hash. +const HEAD_HASH: &str = "0000000000001b62"; + +#[sqlx::test(fixtures("base", "drafts_move_taken"))] +async fn test_rename_onto_a_draft_is_refused(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let client = reqwest::Client::new(); + + let resp = client + .post(format!( + "http://localhost:{port}/api/w/test-workspace/scripts/create" + )) + .header("Authorization", "Bearer SECRET_TOKEN") + .json(&json!({ + "path": "u/test-user/mvtaken_b", + "parent_hash": HEAD_HASH, + "summary": "A", + "description": "", + "content": "export function main() { return 1 }", + "language": "deno", + "schema": {} + })) + .send() + .await?; + let status = resp.status(); + let body = resp.text().await?; + assert_eq!(status, 400, "rename onto a draft was not refused: {body}"); + assert!( + body.contains("already has a draft"), + "unexpected refusal: {body}" + ); + + // The whole deploy rolled back: the source is still the live head, and the + // draft at the destination is untouched. + let head: Value = client + .get(format!( + "http://localhost:{port}/api/w/test-workspace/scripts/get/p/u/test-user/mvtaken_a" + )) + .header("Authorization", "Bearer SECRET_TOKEN") + .send() + .await? + .json() + .await?; + assert_eq!(head["hash"], HEAD_HASH, "source was replaced: {head}"); + assert_eq!(head["archived"], false, "source was archived: {head}"); + + let draft: Value = client + .get(format!("http://localhost:{port}/api/w/test-workspace/drafts/get_own/script/u/test-user/mvtaken_b")) + .header("Authorization", "Bearer SECRET_TOKEN") + .send() + .await? + .json() + .await?; + assert_eq!( + draft["value"]["summary"], "B", + "destination draft changed: {draft}" + ); + + Ok(()) +} + +/// A legacy (ownerless) draft occupies its path too: a deploy there deletes it together +/// with the caller's own row, so a move that parks a second draft beside it would discard +/// edits the caller never saw. Only an admin can clear it, so the refusal says so. +#[sqlx::test(fixtures("base", "drafts_move_taken"))] +async fn test_draft_move_refuses_a_legacy_destination(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + let resp = reqwest::Client::new() + .post(format!( + "http://localhost:{port}/api/w/test-workspace/drafts/move/script/u/test-user/mvtaken_b" + )) + .header("Authorization", "Bearer SECRET_TOKEN") + .json(&json!({ "new_path": "u/test-user/mvtaken_legacy" })) + .send() + .await?; + let status = resp.status(); + let body = resp.text().await?; + assert_eq!(status, 400, "move onto a legacy draft was allowed: {body}"); + assert!( + body.contains("legacy workspace draft") && body.contains("workspace admin"), + "the refusal did not point at the one remedy: {body}" + ); + + // Both rows stayed where they were: the caller's own, and the legacy one the list + // synthesizes under the caller's name. + let list: Vec = reqwest::Client::new() + .get(format!( + "http://localhost:{port}/api/w/test-workspace/drafts/list" + )) + .header("Authorization", "Bearer SECRET_TOKEN") + .send() + .await? + .json() + .await?; + let mut at = list + .iter() + .filter_map(|d| Some((d["kind"].as_str()?, d["path"].as_str()?))) + .filter(|(_, p)| p.starts_with("u/test-user/mvtaken_b") || p.ends_with("mvtaken_legacy")) + .collect::>(); + at.sort(); + assert_eq!( + at, + vec![ + ("script", "u/test-user/mvtaken_b"), + ("script", "u/test-user/mvtaken_legacy") + ], + "{list:?}" + ); + Ok(()) +} + +/// A classic app and a raw app deploy into the same table, so a draft-only move onto +/// the other kind's draft must be refused: deploying either path afterwards deletes +/// the caller's drafts of both kinds, taking the loser's item with it. +#[sqlx::test(fixtures("base", "drafts_move_taken"))] +async fn test_draft_move_refuses_the_other_app_kind(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + let resp = reqwest::Client::new() + .post(format!( + "http://localhost:{port}/api/w/test-workspace/drafts/move/raw_app/u/test-user/mvtaken_raw" + )) + .header("Authorization", "Bearer SECRET_TOKEN") + .json(&json!({ "new_path": "u/test-user/mvtaken_app" })) + .send() + .await?; + let status = resp.status(); + let body = resp.text().await?; + assert_eq!( + status, 400, + "move onto a classic app draft was allowed: {body}" + ); + assert!( + body.contains("already have a draft at 'u/test-user/mvtaken_app' (app)"), + "the refusal did not name the occupying kind: {body}" + ); + + // Both drafts are untouched. + let list: Vec = reqwest::Client::new() + .get(format!( + "http://localhost:{port}/api/w/test-workspace/drafts/list" + )) + .header("Authorization", "Bearer SECRET_TOKEN") + .send() + .await? + .json() + .await?; + let mut at = list + .iter() + .filter(|d| matches!(d["kind"].as_str(), Some("app") | Some("raw_app"))) + .filter_map(|d| Some((d["kind"].as_str()?, d["path"].as_str()?))) + .collect::>(); + at.sort(); + assert_eq!( + at, + vec![ + ("app", "u/test-user/mvtaken_app"), + ("raw_app", "u/test-user/mvtaken_raw") + ], + "{list:?}" + ); + Ok(()) +} + +/// Teammates' drafts of one item share its path by design, so another user's row is no +/// obstacle — except across the app pair, where the two kinds are different items on one +/// deployed path: deploying either strands the other, and deleting the app takes both. +#[sqlx::test(fixtures("base", "drafts_move_taken"))] +async fn test_draft_move_refuses_another_users_other_app_kind( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let move_to = |kind: &'static str, from: &'static str, to: &'static str| async move { + let resp = reqwest::Client::new() + .post(format!( + "http://localhost:{port}/api/w/test-workspace/drafts/move/{kind}/{from}" + )) + .header("Authorization", "Bearer SECRET_TOKEN") + .json(&json!({ "new_path": to })) + .send() + .await?; + Ok::<_, anyhow::Error>((resp.status(), resp.text().await?)) + }; + + let (status, body) = move_to( + "app", + "u/test-user/mvtaken_app", + "u/test-user/mvtaken_theirs", + ) + .await?; + assert_eq!( + status, 400, + "a classic app was moved onto another user's raw app: {body}" + ); + assert!( + body.contains("holds another user's raw app draft"), + "the refusal did not name the occupant: {body}" + ); + + // And the other direction, where the occupant reads as the classic kind. + let (status, body) = move_to( + "raw_app", + "u/test-user/mvtaken_raw", + "u/test-user/mvtaken_app_theirs", + ) + .await?; + assert_eq!( + status, 400, + "a raw app was moved onto another user's classic app: {body}" + ); + assert!( + body.contains("holds another user's app draft"), + "the refusal did not name the occupant: {body}" + ); + + // The same-kind case is the ordinary one: two users' drafts of one raw app. + let (status, body) = move_to( + "raw_app", + "u/test-user/mvtaken_raw", + "u/test-user/mvtaken_theirs", + ) + .await?; + assert!( + status.is_success(), + "a raw app was refused beside another user's raw-app draft: {body}" + ); + Ok(()) +} diff --git a/backend/tests/drafts_nul.rs b/backend/tests/drafts_nul.rs index d7314b8fb6..2a77fe772a 100644 --- a/backend/tests/drafts_nul.rs +++ b/backend/tests/drafts_nul.rs @@ -78,3 +78,46 @@ async fn test_draft_write_strips_nul(db: Pool) -> anyhow::Result<()> { Ok(()) } + +/// The lineage the server derives for `draft.base` comes from the same value: read before +/// the sanitizer, a NUL in it costs the draft its base, which then reads as up to date +/// whatever the head is. +#[sqlx::test(fixtures("drafts_nul"))] +async fn test_draft_base_is_derived_from_sanitized_value(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let base = format!("http://localhost:{port}/api/w/dnul-ws"); + + let resp = authed(client().post(format!( + "{base}/drafts/update/script/u/dnul-admin/poisoned_base" + ))) + .json(&json!({ + "value": { + "summary": "s", + "path": "u/dnul-admin/poisoned_base", + "content": "x", + "parent_hash": "0000000000001b62\u{0}" + } + })) + .send() + .await?; + assert_eq!( + resp.status(), + 200, + "save refused: {}", + resp.text().await.unwrap_or_default() + ); + + let stored: Option = sqlx::query_scalar( + "SELECT base FROM draft WHERE workspace_id = 'dnul-ws' AND path = 'u/dnul-admin/poisoned_base'", + ) + .fetch_one(&db) + .await?; + assert_eq!( + stored.as_deref(), + Some("0000000000001b62"), + "the NUL cost the draft its base: {stored:?}" + ); + Ok(()) +} diff --git a/backend/tests/drafts_save_follows_move.rs b/backend/tests/drafts_save_follows_move.rs new file mode 100644 index 0000000000..afbec56c75 --- /dev/null +++ b/backend/tests/drafts_save_follows_move.rs @@ -0,0 +1,587 @@ +//! A draft save addressed to a path its item moved away from lands on the moved draft. +//! +//! A move carries every draft on the item to the new path and records where they +//! went. An editor left open across it still saves to the old path; the server puts +//! the save on the moved draft, keeps the path keys the move gave it, and names the +//! new path so the editor can follow. Without the record the save would plant a +//! phantom draft-only item at the old location. + +use serde_json::{json, Value}; +use sqlx::{Pool, Postgres}; + +use windmill_test_utils::*; + +/// Hex form of script hash 7030, the way the API takes a parent hash. +const HEAD_HASH: &str = "0000000000001b76"; + +async fn own_draft_value(port: u16, path: &str) -> anyhow::Result { + let draft: Value = reqwest::Client::new() + .get(format!( + "http://localhost:{port}/api/w/test-workspace/drafts/get_own/script/{path}" + )) + .header("Authorization", "Bearer SECRET_TOKEN") + .send() + .await? + .json() + .await?; + Ok(draft["value"].clone()) +} + +async fn own_draft_paths(port: u16) -> anyhow::Result> { + let list: Vec = reqwest::Client::new() + .get(format!( + "http://localhost:{port}/api/w/test-workspace/drafts/list" + )) + .header("Authorization", "Bearer SECRET_TOKEN") + .send() + .await? + .json() + .await?; + Ok(list + .iter() + .filter(|d| d["kind"] == "script") + .filter_map(|d| d["path"].as_str().map(String::from)) + .filter(|p| p.starts_with("u/test-user/follow_")) + .collect()) +} + +#[sqlx::test(fixtures("base", "drafts_save_follows_move"))] +async fn test_save_follows_a_rename(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let client = reqwest::Client::new(); + + // Rename from Home: a redeploy of the deployed content at the new path that + // keeps the deployer's own draft, so the draft is carried rather than consumed. + let resp = client + .post(format!( + "http://localhost:{port}/api/w/test-workspace/scripts/create" + )) + .header("Authorization", "Bearer SECRET_TOKEN") + .json(&json!({ + "path": "u/test-user/follow_b", + "parent_hash": HEAD_HASH, + "summary": "A", + "description": "", + "content": "export function main() { return 1 }", + "language": "deno", + "schema": {}, + "skip_draft_deletion": true + })) + .send() + .await?; + assert_eq!(resp.status(), 201, "rename failed: {}", resp.text().await?); + assert_eq!(own_draft_paths(port).await?, vec!["u/test-user/follow_b"]); + // A script draft's `path` is where deploying it lands, so it moves with the row. + assert_eq!( + own_draft_value(port, "u/test-user/follow_b").await?["path"], + "u/test-user/follow_b" + ); + + // The editor is still on the old path and writes that path back into the value. + let saved: Value = client + .post(format!( + "http://localhost:{port}/api/w/test-workspace/drafts/update/script/u/test-user/follow_a" + )) + .header("Authorization", "Bearer SECRET_TOKEN") + .json(&json!({ + "value": { + "path": "u/test-user/follow_a", + "parent_hash": HEAD_HASH, + "summary": "A", + "content": "edited after the move" + } + })) + .send() + .await? + .json() + .await?; + assert_eq!(saved["status"], "saved", "save refused: {saved}"); + assert_eq!( + saved["path"], "u/test-user/follow_b", + "save did not follow the row: {saved}" + ); + + // The write landed on the carried row; nothing reappeared at the old path. + assert_eq!(own_draft_paths(port).await?, vec!["u/test-user/follow_b"]); + let draft = own_draft_value(port, "u/test-user/follow_b").await?; + assert_eq!(draft["content"], "edited after the move", "{draft}"); + assert_eq!(draft["path"], "u/test-user/follow_b", "{draft}"); + + Ok(()) +} + +/// A draft-only move rewrites both path keys. The owner's open editor still carries +/// the typed path it had, which names neither the old nor the new path; the moved +/// draft's own keys have to win, or the save walks the item back. +#[sqlx::test(fixtures("base", "drafts_save_follows_move"))] +async fn test_save_follows_a_draft_only_move(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let client = reqwest::Client::new(); + + let resp = client + .post(format!( + "http://localhost:{port}/api/w/test-workspace/drafts/move/script/u/test-user/draft_store" + )) + .header("Authorization", "Bearer SECRET_TOKEN") + .json(&json!({ "new_path": "u/test-user/moved" })) + .send() + .await?; + assert!( + resp.status().is_success(), + "move failed: {}", + resp.text().await? + ); + + let saved: Value = client + .post(format!( + "http://localhost:{port}/api/w/test-workspace/drafts/update/script/u/test-user/draft_store" + )) + .header("Authorization", "Bearer SECRET_TOKEN") + .json(&json!({ + "value": { + "path": "u/test-user/friendly", + "draft_path": "u/test-user/friendly", + "summary": "D", + "content": "edited after the move" + } + })) + .send() + .await? + .json() + .await?; + assert_eq!(saved["path"], "u/test-user/moved", "{saved}"); + + let draft = own_draft_value(port, "u/test-user/moved").await?; + assert_eq!(draft["content"], "edited after the move", "{draft}"); + assert_eq!(draft["path"], "u/test-user/moved", "{draft}"); + assert_eq!(draft["draft_path"], "u/test-user/moved", "{draft}"); + Ok(()) +} + +/// Rename `from` to `to` the way Home does: redeploy the deployed content at the new +/// path, keeping the deployer's own draft so it is carried rather than consumed. +/// Returns the new head's hash. +async fn rename(port: u16, from_hash: &str, to: &str) -> anyhow::Result { + let resp = reqwest::Client::new() + .post(format!( + "http://localhost:{port}/api/w/test-workspace/scripts/create" + )) + .header("Authorization", "Bearer SECRET_TOKEN") + .json(&json!({ + "path": to, + "parent_hash": from_hash, + "summary": "A", + "description": "", + "content": "export function main() { return 1 }", + "language": "deno", + "schema": {}, + "skip_draft_deletion": true + })) + .send() + .await?; + let status = resp.status(); + let hash = resp.text().await?; + assert_eq!(status, 201, "rename to {to} failed: {hash}"); + Ok(hash) +} + +/// Save the draft as an editor still bound to `url_path` would. Returns the path the +/// save landed at. +async fn save_at(port: u16, url_path: &str, content: &str) -> anyhow::Result { + let saved: Value = reqwest::Client::new() + .post(format!( + "http://localhost:{port}/api/w/test-workspace/drafts/update/script/{url_path}" + )) + .header("Authorization", "Bearer SECRET_TOKEN") + .json(&json!({ + "value": { "path": url_path, "summary": "A", "content": content, "language": "deno" } + })) + .send() + .await? + .json() + .await?; + assert_eq!(saved["status"], "saved", "save refused: {saved}"); + Ok(saved["path"].as_str().unwrap_or_default().to_string()) +} + +/// A record is kept to one hop, and a move back to the path it left ends it: both are +/// three statements whose order decides the answer. +#[sqlx::test(fixtures("base", "drafts_save_follows_move"))] +async fn test_move_records_stay_one_hop(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + let b = rename(port, HEAD_HASH, "u/test-user/follow_b").await?; + let _c = rename(port, &b, "u/test-user/follow_c").await?; + assert_eq!( + save_at(port, "u/test-user/follow_a", "after two moves").await?, + "u/test-user/follow_c", + "a save at the first path did not reach the last" + ); + assert_eq!(own_draft_paths(port).await?, vec!["u/test-user/follow_c"]); + Ok(()) +} + +#[sqlx::test(fixtures("base", "drafts_save_follows_move"))] +async fn test_move_back_ends_the_record(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + let b = rename(port, HEAD_HASH, "u/test-user/follow_b").await?; + let _a = rename(port, &b, "u/test-user/follow_a").await?; + assert_eq!( + save_at(port, "u/test-user/follow_a", "after moving back").await?, + "u/test-user/follow_a", + "a save was routed off the path the item moved back to" + ); + assert_eq!(own_draft_paths(port).await?, vec!["u/test-user/follow_a"]); + Ok(()) +} + +/// A rename carries a teammate's row too: both its path keys follow, and the version +/// it forked from does not move. A restamp there would clear their out-of-date prompt +/// and let them deploy over the mover's version believing they were current. +#[sqlx::test(fixtures("base", "drafts_save_follows_move"))] +async fn test_a_teammates_draft_follows_with_its_base(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + rename(port, HEAD_HASH, "u/test-user/follow_b").await?; + + // Read from the pool: the teammate's row is another user's, and this asserts on + // `base`, which no endpoint exposes for someone else's draft. + let row: (String, String, Option) = sqlx::query_as( + "SELECT value::jsonb ->> 'path', value::jsonb ->> 'draft_path', base + FROM draft WHERE workspace_id = 'test-workspace' AND typ = 'script' + AND email = 'test2@windmill.dev'", + ) + .fetch_one(&db) + .await?; + assert_eq!(row.0, "u/test-user/follow_b", "typed path did not follow"); + assert_eq!(row.1, "u/test-user/follow_b", "mirror did not follow"); + assert_eq!( + row.2.as_deref(), + Some(HEAD_HASH), + "the teammate's base was restamped by someone else's rename" + ); + Ok(()) +} + +/// An item move and then the owner's own move of what is left: the two records have +/// different scopes, so the owner's move has to extend the chain in its own scope or +/// a save addressed to the first path stops at the abandoned middle one. +#[sqlx::test(fixtures("base", "drafts_save_follows_move"))] +async fn test_an_owner_move_extends_an_item_move(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + rename(port, HEAD_HASH, "u/test-user/follow_b").await?; + // Archiving the script at the new path leaves the carried draft as a draft-only + // item, which its owner can move through `/drafts/move`. + sqlx::query("UPDATE script SET archived = true WHERE path = 'u/test-user/follow_b'") + .execute(&db) + .await?; + let resp = reqwest::Client::new() + .post(format!( + "http://localhost:{port}/api/w/test-workspace/drafts/move/script/u/test-user/follow_b" + )) + .header("Authorization", "Bearer SECRET_TOKEN") + .json(&json!({ "new_path": "u/test-user/follow_c" })) + .send() + .await?; + assert!( + resp.status().is_success(), + "move failed: {}", + resp.text().await? + ); + + assert_eq!( + save_at(port, "u/test-user/follow_a", "after both moves").await?, + "u/test-user/follow_c", + "a save at the first path stopped at the path the owner's move left" + ); + assert_eq!(own_draft_paths(port).await?, vec!["u/test-user/follow_c"]); + Ok(()) +} + +/// Redeploying at a path an owner's move routed away from ends that route: the live item +/// owns its path again, and its saves must not follow the draft that left. +#[sqlx::test(fixtures("base", "drafts_save_follows_move"))] +async fn test_redeploy_at_a_routed_path_ends_the_route(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + // `move_draft` ignores archived rows, so an archived script's draft can be moved away. + sqlx::query("UPDATE script SET archived = true WHERE path = 'u/test-user/follow_a'") + .execute(&db) + .await?; + let resp = reqwest::Client::new() + .post(format!( + "http://localhost:{port}/api/w/test-workspace/drafts/move/script/u/test-user/follow_a" + )) + .header("Authorization", "Bearer SECRET_TOKEN") + .json(&json!({ "new_path": "u/test-user/follow_b" })) + .send() + .await?; + assert!( + resp.status().is_success(), + "move failed: {}", + resp.text().await? + ); + + // Unarchiving redeploys at the same path, with the archived version as parent. + rename(port, HEAD_HASH, "u/test-user/follow_a").await?; + + assert_eq!( + save_at(port, "u/test-user/follow_a", "for the live script").await?, + "u/test-user/follow_a", + "a save for the redeployed script followed the draft that moved away" + ); + Ok(()) +} + +/// A draft written before the NUL sanitizer still has to follow a move: its path keys are +/// what a deploy of it would land on, so the carry rewrites them, sanitizing the value it +/// could not otherwise parse. +#[sqlx::test(fixtures("base", "drafts_save_follows_move"))] +async fn test_a_poisoned_draft_follows_a_rename(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + // The teammate's row, rewritten the way a pre-sanitizer client left one: a real NUL + // escape in the content, both path keys naming the path the item is about to leave. + sqlx::query( + r#"UPDATE draft SET value = '{"path": "u/test-user/follow_a", "draft_path": "u/test-user/follow_a", + "parent_hash": "0000000000001b76", "summary": "A", "content": "a\u0000b"}' + WHERE email = 'test2@windmill.dev'"#, + ) + .execute(&db) + .await?; + + rename(port, HEAD_HASH, "u/test-user/follow_b").await?; + + let row: (String, String, String) = sqlx::query_as( + "SELECT value::jsonb ->> 'path', value::jsonb ->> 'draft_path', value::jsonb ->> 'content' + FROM draft WHERE email = 'test2@windmill.dev'", + ) + .fetch_one(&db) + .await?; + assert_eq!(row.0, "u/test-user/follow_b", "typed path did not follow"); + assert_eq!(row.1, "u/test-user/follow_b", "mirror did not follow"); + assert_eq!(row.2, "ab", "the NUL survived the rewrite"); + Ok(()) +} + +/// A discard whose answer is lost is retried, and the retry finds nothing to delete. It +/// still has to name where the item went: the editor asking is on the path it left, and a +/// reload there lands on nothing. +#[sqlx::test(fixtures("base", "drafts_save_follows_move"))] +async fn test_a_retried_discard_still_names_the_destination( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let client = reqwest::Client::new(); + + rename(port, HEAD_HASH, "u/test-user/follow_b").await?; + + let discard = |token: &'static str| { + let client = client.clone(); + async move { + client + .post(format!( + "http://localhost:{port}/api/w/test-workspace/drafts/update/script/u/test-user/follow_a" + )) + .header("Authorization", format!("Bearer {token}")) + .json(&json!({ "value": null })) + .send() + .await? + .json::() + .await + } + }; + + let first = discard("SECRET_TOKEN").await?; + assert_eq!(first["status"], "saved", "{first}"); + assert_eq!(first["path"], "u/test-user/follow_b", "{first}"); + + let retry = discard("SECRET_TOKEN").await?; + assert_eq!( + retry["status"], "saved", + "the retry was not idempotent: {retry}" + ); + assert_eq!(retry["path"], "u/test-user/follow_b", "{retry}"); + + // A third user has no draft on this item, so their discard deletes nothing and the + // destination is a move record and nothing else: it is answered only to a caller who + // can read that path. + let other = discard("SECRET_TOKEN_3").await?; + assert_eq!(other["status"], "saved", "{other}"); + assert_eq!( + other["path"], + Value::Null, + "a user who cannot read the destination was told where the item went: {other}" + ); + Ok(()) +} + +/// The legacy workspace-level row is carried by a rename like any other draft, and the +/// record that routes saves to it covers every caller — so discarding it from a page that +/// still names the old path has to reach it where it went. +#[sqlx::test(fixtures("base", "drafts_save_follows_move"))] +async fn test_a_legacy_discard_follows_a_rename(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + sqlx::query( + r#"INSERT INTO draft (workspace_id, path, typ, value, email) + VALUES ('test-workspace', 'u/test-user/follow_a', 'script', + '{"path": "u/test-user/follow_a", "summary": "legacy", "content": "x"}', NULL)"#, + ) + .execute(&db) + .await?; + + rename(port, HEAD_HASH, "u/test-user/follow_b").await?; + + let resp = reqwest::Client::new() + .post(format!( + "http://localhost:{port}/api/w/test-workspace/drafts/update/script/u/test-user/follow_a" + )) + .header("Authorization", "Bearer SECRET_TOKEN") + .json(&json!({ "value": null, "legacy": true })) + .send() + .await?; + assert!( + resp.status().is_success(), + "discard failed: {}", + resp.text().await? + ); + + let left: i64 = sqlx::query_scalar( + "SELECT count(*) FROM draft WHERE workspace_id = 'test-workspace' AND email IS NULL", + ) + .fetch_one(&db) + .await?; + assert_eq!( + left, 0, + "the legacy draft survived a discard aimed at its old path" + ); + Ok(()) +} + +/// Deploying a draft that a move carried off an archived script: its parent is still the +/// version at the old path, so the deploy renames from there and carries what is left over +/// — onto the very draft being deployed. That row is this deploy's own, not an item in its +/// way, or the deploy is refused and every retry refuses again. +#[sqlx::test(fixtures("base", "drafts_save_follows_move"))] +async fn test_a_moved_draft_deploys_at_its_new_path(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + sqlx::query("UPDATE script SET archived = true WHERE path = 'u/test-user/follow_a'") + .execute(&db) + .await?; + let resp = reqwest::Client::new() + .post(format!( + "http://localhost:{port}/api/w/test-workspace/drafts/move/script/u/test-user/follow_a" + )) + .header("Authorization", "Bearer SECRET_TOKEN") + .json(&json!({ "new_path": "u/test-user/follow_b" })) + .send() + .await?; + assert!( + resp.status().is_success(), + "move failed: {}", + resp.text().await? + ); + + // Deploy it where it now lives, still parented on the archived version it forked from. + let resp = reqwest::Client::new() + .post(format!( + "http://localhost:{port}/api/w/test-workspace/scripts/create" + )) + .header("Authorization", "Bearer SECRET_TOKEN") + .json(&json!({ + "path": "u/test-user/follow_b", + "parent_hash": HEAD_HASH, + "summary": "A", + "description": "", + "content": "export function main() { return 2 }", + "language": "deno", + "schema": {} + })) + .send() + .await?; + let status = resp.status(); + let body = resp.text().await?; + assert_eq!(status, 201, "the moved draft could not be deployed: {body}"); + Ok(()) +} + +/// A route is only as good as the item it points at: when an unrelated item claims the +/// destination, a save still addressed to the old path must stay where it is rather than +/// land on that item's draft. +#[sqlx::test(fixtures("base", "drafts_save_follows_move"))] +async fn test_a_reused_destination_ends_the_route(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + sqlx::query("UPDATE script SET archived = true WHERE path = 'u/test-user/follow_a'") + .execute(&db) + .await?; + let resp = reqwest::Client::new() + .post(format!( + "http://localhost:{port}/api/w/test-workspace/drafts/move/script/u/test-user/follow_a" + )) + .header("Authorization", "Bearer SECRET_TOKEN") + .json(&json!({ "new_path": "u/test-user/follow_b" })) + .send() + .await?; + assert!( + resp.status().is_success(), + "move failed: {}", + resp.text().await? + ); + // Someone else's item takes the destination, and the moved draft goes with the + // deploy that consumes it. + sqlx::query("DELETE FROM draft WHERE workspace_id = 'test-workspace' AND path = 'u/test-user/follow_b'") + .execute(&db) + .await?; + let resp = reqwest::Client::new() + .post(format!( + "http://localhost:{port}/api/w/test-workspace/scripts/create" + )) + .header("Authorization", "Bearer SECRET_TOKEN") + .json(&json!({ + "path": "u/test-user/follow_b", + "summary": "unrelated", + "description": "", + "content": "export function main() { return 3 }", + "language": "deno", + "schema": {} + })) + .send() + .await?; + assert_eq!(resp.status(), 201, "create failed: {}", resp.text().await?); + + assert_eq!( + save_at(port, "u/test-user/follow_a", "after the destination was reused").await?, + "u/test-user/follow_a", + "a save was routed onto the item that now owns the destination" + ); + Ok(()) +} diff --git a/backend/tests/fixtures/drafts_move_mirror.sql b/backend/tests/fixtures/drafts_move_mirror.sql new file mode 100644 index 0000000000..e96d985eae --- /dev/null +++ b/backend/tests/fixtures/drafts_move_mirror.sql @@ -0,0 +1,16 @@ +-- Fixture for the two-path-key move test. +-- +-- Two draft-only scripts owned by test-user, both parked at a generated storage +-- key the way a session-created draft is. One carries the `draft_path` mirror the +-- session editor writes while the typed path differs from that key; the other +-- carries no mirror at all, which is what a plain draft looks like. + +INSERT INTO draft (workspace_id, path, typ, value, email) VALUES + ('test-workspace', 'u/test-user/draft_mirror', 'script', + '{"path": "u/test-user/friendly", "draft_path": "u/test-user/friendly", + "content": "x", "language": "bun", "summary": "S"}', + 'test@windmill.dev'), + ('test-workspace', 'u/test-user/draft_plain', 'script', + '{"path": "u/test-user/draft_plain", + "content": "x", "language": "bun", "summary": "S"}', + 'test@windmill.dev'); diff --git a/backend/tests/fixtures/drafts_move_taken.sql b/backend/tests/fixtures/drafts_move_taken.sql new file mode 100644 index 0000000000..5fee11b942 --- /dev/null +++ b/backend/tests/fixtures/drafts_move_taken.sql @@ -0,0 +1,37 @@ +-- Fixture for refusing a rename onto a path a draft already occupies. +-- +-- A deployed script at `u/test-user/mvtaken_a` (hash 7010 = 0x1b62), and a +-- never-deployed draft of test-user's own at `u/test-user/mvtaken_b`, the path +-- the rename will target. Nothing deployed lives at the target, so only the +-- draft can refuse the move. + +INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by, + schema, summary, description, lock, extra_perms) +VALUES ('test-workspace', 7010, 'u/test-user/mvtaken_a', + 'export function main() { return 1 }', + 'deno', 'script', 'test-user', '{}', 'A', '', '', '{}'); + +INSERT INTO draft (workspace_id, path, typ, value, email) +VALUES ('test-workspace', 'u/test-user/mvtaken_b', 'script', + '{"path": "u/test-user/mvtaken_b", "summary": "B", "content": ""}', + 'test@windmill.dev'); + +-- An ownerless (pre per-user drafts) row at a third path: a deploy there deletes it +-- together with the caller's, so it occupies the path just as an own draft does. +INSERT INTO draft (workspace_id, path, typ, value, email) +VALUES ('test-workspace', 'u/test-user/mvtaken_legacy', 'script', + '{"path": "u/test-user/mvtaken_legacy", "summary": "legacy", "content": ""}', + NULL); + +-- A draft-only classic app and a draft-only raw app of the same owner. They share +-- the `app` table, so one occupies the other's path. The third is another user's raw +-- app: a different item at its own path, not a second copy of anyone's. +INSERT INTO draft (workspace_id, path, typ, value, email) VALUES + ('test-workspace', 'u/test-user/mvtaken_app', 'app', + '{"summary": "classic", "value": {}}', 'test@windmill.dev'), + ('test-workspace', 'u/test-user/mvtaken_raw', 'raw_app', + '{"summary": "raw", "files": {}}', 'test@windmill.dev'), + ('test-workspace', 'u/test-user/mvtaken_theirs', 'raw_app', + '{"summary": "theirs", "files": {}}', 'test2@windmill.dev'), + ('test-workspace', 'u/test-user/mvtaken_app_theirs', 'app', + '{"summary": "theirs", "value": {}}', 'test2@windmill.dev'); diff --git a/backend/tests/fixtures/drafts_save_follows_move.sql b/backend/tests/fixtures/drafts_save_follows_move.sql new file mode 100644 index 0000000000..16a41feb1a --- /dev/null +++ b/backend/tests/fixtures/drafts_save_follows_move.sql @@ -0,0 +1,29 @@ +-- Fixture for saving a draft to the path its item moved away from. +-- +-- A deployed script at `u/test-user/follow_a` (hash 7030 = 0x1b76) with the +-- deployer's own draft on it. The test renames the script and then saves the +-- draft from an editor still bound to the old path. + +INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by, + schema, summary, description, lock, extra_perms) +VALUES ('test-workspace', 7030, 'u/test-user/follow_a', + 'export function main() { return 1 }', + 'deno', 'script', 'test-user', '{}', 'A', '', '', '{}'); + +INSERT INTO draft (workspace_id, path, typ, value, email, base) +VALUES ('test-workspace', 'u/test-user/follow_a', 'script', + '{"path": "u/test-user/follow_a", "parent_hash": "0000000000001b76", "summary": "A", "content": "draft"}', + 'test@windmill.dev', '0000000000001b76'); + +-- A draft-only script parked at a generated storage key, its typed path elsewhere. +INSERT INTO draft (workspace_id, path, typ, value, email) +VALUES ('test-workspace', 'u/test-user/draft_store', 'script', + '{"path": "u/test-user/friendly", "draft_path": "u/test-user/friendly", "summary": "D", "content": "draft"}', + 'test@windmill.dev'); + +-- A teammate's draft on the same deployed script, forked from the same head. The +-- rename must carry it too, without touching the version it forked from. +INSERT INTO draft (workspace_id, path, typ, value, email, base) +VALUES ('test-workspace', 'u/test-user/follow_a', 'script', + '{"path": "u/test-user/follow_a", "draft_path": "u/test-user/follow_a", "parent_hash": "0000000000001b76", "summary": "A", "content": "teammate draft"}', + 'test2@windmill.dev', '0000000000001b76'); diff --git a/backend/tests/perpetual_cancel_race.rs b/backend/tests/perpetual_cancel_race.rs new file mode 100644 index 0000000000..df239eb7ab --- /dev/null +++ b/backend/tests/perpetual_cancel_race.rs @@ -0,0 +1,183 @@ +//! A cancel reaches the queue row first and the worker only later: one landing after the worker's +//! last read of that row completes the run carrying no `canceled_by` of its own, and a perpetual +//! script must still stop rather than restart as if nothing had been cancelled. + +use serde_json::json; +use sqlx::{types::Json, Pool, Postgres}; +use uuid::Uuid; +use windmill_queue::{add_completed_job, get_mini_completed_job}; + +const W_ID: &str = "test-workspace"; +const PATH: &str = "u/test-user/loop"; + +async fn start_perpetual_run(db: &Pool) -> anyhow::Result { + sqlx::query( + "INSERT INTO script (workspace_id, hash, path, summary, description, content, created_by, \ + language, lock, restart_unless_cancelled) \ + VALUES ($1, 1, $2, '', '', 'echo', 'test-user', 'bash', '', true)", + ) + .bind(W_ID) + .bind(PATH) + .execute(db) + .await?; + let id = Uuid::new_v4(); + sqlx::query( + "INSERT INTO v2_job (id, workspace_id, created_by, created_at, permissioned_as, \ + permissioned_as_email, kind, runnable_id, runnable_path, script_lang, tag, args, \ + visible_to_owner) \ + VALUES ($1, $2, 'test-user', now(), 'u/test-user', 'test@windmill.dev', 'script', 1, \ + $3, 'bash', 'bash', '{\"n\": 1}', true)", + ) + .bind(id) + .bind(W_ID) + .bind(PATH) + .execute(db) + .await?; + sqlx::query( + "INSERT INTO v2_job_queue (id, workspace_id, scheduled_for, running, started_at, tag) \ + VALUES ($1, $2, now(), true, now(), 'bash')", + ) + .bind(id) + .bind(W_ID) + .execute(db) + .await?; + Ok(id) +} + +/// Waits for another connection to be stuck on the queue row of `id`, which is where the +/// completion's delete lands while the cancel is uncommitted. +async fn wait_until_blocked_on(db: &Pool, id: Uuid) -> anyhow::Result<()> { + for _ in 0..100 { + let blocked: bool = sqlx::query_scalar( + "SELECT EXISTS (SELECT 1 FROM pg_locks l JOIN v2_job_queue q \ + ON q.ctid = ('(' || l.page || ',' || l.tuple || ')')::tid \ + WHERE NOT l.granted AND l.locktype = 'tuple' AND q.id = $1) \ + OR EXISTS (SELECT 1 FROM pg_stat_activity \ + WHERE wait_event_type = 'Lock' AND query LIKE '%v2_job_queue%')", + ) + .bind(id) + .fetch_one(db) + .await?; + if blocked { + return Ok(()); + } + tokio::time::sleep(std::time::Duration::from_millis(50)).await; + } + anyhow::bail!("the completion never reached the delete, so the race was not exercised") +} + +/// The same cancel, one step earlier: still being written when the completion reads the queue row, +/// which reads it without a lock. Only the delete that follows waits for the writer. +#[sqlx::test(fixtures("base"))] +async fn a_cancel_still_being_written_is_kept(db: Pool) -> anyhow::Result<()> { + let running = start_perpetual_run(&db).await?; + let job = get_mini_completed_job(&running, W_ID, &db).await?.unwrap(); + + let mut cancel = db.begin().await?; + sqlx::query( + "UPDATE v2_job_queue SET canceled_by = 'test-user', canceled_reason = 'stop' WHERE id = $1", + ) + .bind(running) + .execute(&mut *cancel) + .await?; + + let completing = tokio::spawn({ + let db = db.clone(); + async move { + add_completed_job( + &db, + &job, + true, + false, + Json(&json!("done")), + None, + 0, + None, + false, + None, + false, + ) + .await + } + }); + // Committing on a timer would let a slow completion read the cancel through the insert + // instead, leaving the half this test is for unexercised and green. Waiting for the delete to + // block on the row is what puts it there. + wait_until_blocked_on(&db, running).await?; + cancel.commit().await?; + completing.await??; + + let (status, canceled_by): (String, Option) = + sqlx::query_as("SELECT status::text, canceled_by FROM v2_job_completed WHERE id = $1") + .bind(running) + .fetch_one(&db) + .await?; + assert_eq!(status, "canceled", "the run is recorded as canceled"); + assert_eq!(canceled_by.as_deref(), Some("test-user")); + + let queued: Vec = sqlx::query_scalar( + "SELECT q.id FROM v2_job_queue q JOIN v2_job j USING (id) \ + WHERE j.workspace_id = $1 AND j.runnable_path = $2", + ) + .bind(W_ID) + .bind(PATH) + .fetch_all(&db) + .await?; + assert!( + queued.is_empty(), + "a canceled perpetual run queues no next one: {queued:?}" + ); + Ok(()) +} + +#[sqlx::test(fixtures("base"))] +async fn a_cancel_landing_after_the_worker_read_its_row_is_kept( + db: Pool, +) -> anyhow::Result<()> { + let running = start_perpetual_run(&db).await?; + sqlx::query( + "UPDATE v2_job_queue SET canceled_by = 'test-user', canceled_reason = 'stop' WHERE id = $1", + ) + .bind(running) + .execute(&db) + .await?; + + let job = get_mini_completed_job(&running, W_ID, &db).await?.unwrap(); + add_completed_job( + &db, + &job, + true, + false, + Json(&json!("done")), + None, + 0, + // What the worker carries when the cancel landed after it last read the row. + None, + false, + None, + false, + ) + .await?; + + let (status, canceled_by): (String, Option) = + sqlx::query_as("SELECT status::text, canceled_by FROM v2_job_completed WHERE id = $1") + .bind(running) + .fetch_one(&db) + .await?; + assert_eq!(status, "canceled", "the run is recorded as canceled"); + assert_eq!(canceled_by.as_deref(), Some("test-user")); + + let queued: Vec = sqlx::query_scalar( + "SELECT q.id FROM v2_job_queue q JOIN v2_job j USING (id) \ + WHERE j.workspace_id = $1 AND j.runnable_path = $2", + ) + .bind(W_ID) + .bind(PATH) + .fetch_all(&db) + .await?; + assert!( + queued.is_empty(), + "a canceled perpetual run queues no next one: {queued:?}" + ); + Ok(()) +} diff --git a/backend/tests/raw_apps_list_scope.rs b/backend/tests/raw_apps_list_scope.rs new file mode 100644 index 0000000000..503bca7577 --- /dev/null +++ b/backend/tests/raw_apps_list_scope.rs @@ -0,0 +1,76 @@ +//! `GET /raw_apps/list` must honor a path-scoped token. The route layer only checks +//! the scope domain and RLS knows nothing of token scopes, so the per-path filter +//! lives in the handler, ahead of the page's LIMIT. + +use sqlx::{Pool, Postgres}; +use windmill_test_utils::*; + +async fn list_paths(port: u16, token: &str, query: &str) -> anyhow::Result> { + let resp = reqwest::Client::new() + .get(format!( + "http://localhost:{port}/api/w/test-workspace/raw_apps/list?{query}" + )) + .header("Authorization", format!("Bearer {token}")) + .send() + .await?; + assert_eq!(resp.status(), 200, "list: {}", resp.text().await?); + let body: Vec = resp.json().await?; + let mut paths: Vec = body + .iter() + .map(|a| a["path"].as_str().unwrap().to_string()) + .collect(); + paths.sort(); + Ok(paths) +} + +#[sqlx::test(fixtures("base"))] +async fn test_raw_apps_list_filters_by_token_scope(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + + // The list sorts by edited_at desc, so the out-of-scope apps fill the first page + // unless the scope is applied before the LIMIT. + for (path, age_minutes) in [ + ("f/other/c", 0), + ("u/test-userx/d", 1), + ("u/test-user/a", 2), + ("u/test-user/sub/b", 3), + // A `prefix/*` grant also covers the path at the prefix itself. + ("u/test-user", 4), + ] { + sqlx::query( + "INSERT INTO raw_app (path, workspace_id, data, edited_at) + VALUES ($1, 'test-workspace', '', now() - make_interval(mins => $2))", + ) + .bind(path) + .bind(age_minutes) + .execute(&db) + .await?; + } + sqlx::query( + "INSERT INTO token (token_hash, token_prefix, token, email, label, scopes) + VALUES (encode(sha256('SCOPED_TOKEN'::bytea), 'hex'), 'SCOPED_TOK', 'SCOPED_TOKEN', + 'test@windmill.dev', 'scoped', ARRAY['raw_apps:read:u/test-user/*'])", + ) + .execute(&db) + .await?; + + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + // The same user through an unscoped token sees every row. + assert_eq!( + list_paths(port, "SECRET_TOKEN", "").await?, + [ + "f/other/c", + "u/test-user", + "u/test-user/a", + "u/test-user/sub/b", + "u/test-userx/d" + ] + ); + assert_eq!( + list_paths(port, "SCOPED_TOKEN", "per_page=3").await?, + ["u/test-user", "u/test-user/a", "u/test-user/sub/b"] + ); + Ok(()) +} diff --git a/backend/tests/worker.rs b/backend/tests/worker.rs index b8bd6b30a7..d7fb992280 100644 --- a/backend/tests/worker.rs +++ b/backend/tests/worker.rs @@ -5439,7 +5439,7 @@ async fn test_duckdb_ffi(db: Pool) -> anyhow::Result<()> { } /// Test that flow substeps with tags that are not available for the workspace fail. -/// This validates that `check_tag_available_for_workspace_internal` is properly called +/// This validates that `check_tag_available_for_push` is properly called /// when pushing jobs from worker_flow. #[sqlx::test(fixtures("base"))] #[serial] @@ -5512,17 +5512,178 @@ async fn test_flow_substep_tag_availability_check(db: Pool) -> anyhow: Ok(()) } +/// A step's `$flow_expr[...]` tag is checked on the queue it resolves to, not on its text: the +/// custom tags list `bun` alone, which admits the step resolving to it and refuses the one that +/// lands elsewhere. +#[cfg(feature = "deno_core")] +#[sqlx::test(fixtures("base"))] +#[serial] +async fn test_flow_substep_tag_checked_on_resolved_value(db: Pool) -> anyhow::Result<()> { + use windmill_common::worker::{CustomTags, CUSTOM_TAGS_PER_WORKSPACE}; + + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + + CUSTOM_TAGS_PER_WORKSPACE.store(std::sync::Arc::new(CustomTags::from(vec![ + "bun".to_string() + ]))); + + let step = |id: &str, tag: Option<&str>| { + json!({ + "id": id, + "value": { + "type": "rawscript", + "language": "deno", + "content": "export function main() { return { lang: 'bun' } }", + "tag": tag, + }, + }) + }; + let flow: FlowValue = serde_json::from_value(json!({ + "modules": [ + step("a", None), + step("b", Some("$flow_expr[results.a.lang]")), + step("c", Some("$flow_expr[results.a.lang]-gpu")), + ], + }))?; + + // A non-superadmin, so the custom tags apply. + let job = RunJob::from(JobPayload::RawFlow { value: flow, path: None, restarted_from: None }) + .as_user("test-user-2", "test2@windmill.dev") + .run_until_complete(&db, false, server.addr.port()) + .await; + CUSTOM_TAGS_PER_WORKSPACE.store(std::sync::Arc::new(CustomTags::default())); + + let b_tag = sqlx::query_scalar::<_, String>( + "SELECT tag FROM v2_job WHERE parent_job = $1 AND flow_step_id = 'b'", + ) + .bind(job.id) + .fetch_one(&db) + .await?; + assert_eq!(b_tag, "bun"); + + assert!(!job.success); + let result = job.json_result().unwrap(); + let message = result["error"]["message"].as_str().unwrap_or_default(); + assert!( + message.contains("(resolved to bun-gpu) is not included in the allowed CUSTOM_TAGS"), + "got {result:?}" + ); + + Ok(()) +} + +/// A flow whose preprocessor runs lands on its tag filled in from the preprocessor's output, so +/// running it judges the tag as written: even raw args that fill it in to a listed tag do not +/// admit a template nobody listed. +#[cfg(feature = "deno_core")] +#[sqlx::test(fixtures("base", "hello"))] +#[serial] +async fn test_flow_tag_judged_as_written_before_preprocessor( + db: Pool, +) -> anyhow::Result<()> { + use windmill_common::worker::{CustomTags, CUSTOM_TAGS_PER_WORKSPACE}; + + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + let set_flow_tag = |tag: &'static str| { + let db = db.clone(); + async move { + sqlx::query("UPDATE flow SET tag = $1 WHERE path = 'f/system/hello_with_preprocessor'") + .bind(tag) + .execute(&db) + .await?; + // Flow info is cached per version, which a real redeploy would bump. + windmill_common::FLOW_INFO_CACHE + .remove(&("test-workspace".to_string(), 1443253234253456)); + anyhow::Ok(()) + } + }; + // As a non-superadmin, so the custom tags apply. + let run_as_user = |entries: &[&str], query: &str| { + CUSTOM_TAGS_PER_WORKSPACE.store(std::sync::Arc::new(CustomTags::from( + entries.iter().map(|e| e.to_string()).collect(), + ))); + reqwest::Client::new() + .post(format!( + "http://localhost:{port}/api/w/test-workspace/jobs/run/f/f/system/hello_with_preprocessor{query}" + )) + .bearer_auth("SECRET_TOKEN_2") + .json(&json!({ "foo": "bar" })) + .send() + }; + + sqlx::query( + "UPDATE flow SET extra_perms = '{\"u/test-user-2\": false}' + WHERE path = 'f/system/hello_with_preprocessor'", + ) + .execute(&db) + .await?; + set_flow_tag("pp-$args[foo]").await?; + let refused = run_as_user(&["pp-bar"], "").await?; + let refused_status = refused.status(); + let refused_body = refused.text().await?; + // `push` drops a `?tag=` of such a flow, so it does not stand in for the flow's own tag. + let overridden = run_as_user(&["pp-bar"], "?tag=pp-bar").await?.status(); + let admitted = run_as_user(&["pp-$args[foo]"], "").await?.status(); + // A tag the preprocessor cannot change is known now, so patterns apply to it. + set_flow_tag("pp-large").await?; + let static_admitted = run_as_user(&["pp-$args[size]"], "").await?.status(); + + // Nothing a flow's tag can read resolves `$flow_expr[...]`, so the flow keeps its tag. + set_flow_tag("$flow_expr[flow_input.foo]").await?; + CUSTOM_TAGS_PER_WORKSPACE.store(std::sync::Arc::new(CustomTags::default())); + let unresolvable = RunJob::from(JobPayload::Flow { + path: "f/system/hello_with_preprocessor".to_string(), + dedicated_worker: None, + apply_preprocessor: true, + version: 1443253234253456, + labels: None, + }) + .as_user("test-user-2", "test2@windmill.dev") + .run_until_complete(&db, false, port) + .await; + + assert_eq!(refused_status, 400, "got {refused_body}"); + assert!( + refused_body.contains("Tag pp-$args[foo] is not included"), + "got {refused_body}" + ); + assert_eq!(overridden, 400); + assert!(admitted.is_success(), "got {admitted}"); + assert!(static_admitted.is_success(), "got {static_admitted}"); + + assert!(unresolvable.success, "got {:?}", unresolvable.json_result()); + let tag = sqlx::query_scalar::<_, String>("SELECT tag FROM v2_job WHERE id = $1") + .bind(unresolvable.id) + .fetch_one(&db) + .await?; + assert_eq!(tag, "flow"); + + Ok(()) +} + /// The `*` fork marker only grants through a real `parent_workspace_id` lineage lookup, which the /// parse-level unit tests cannot reach: they hand `applies_to_workspace` a synthetic chain, so a /// regression in the lookup or in the `is_fork_scoped()` gate that skips it would pass them. #[sqlx::test(fixtures("base"))] #[serial] async fn test_fork_marker_tag_admission_through_lineage(db: Pool) -> anyhow::Result<()> { - use windmill_common::jobs::check_tag_available_for_workspace_internal; use windmill_common::worker::{CustomTags, CUSTOM_TAGS_PER_WORKSPACE}; initialize_tracing().await; + // For a non-superadmin caller (a superadmin would bypass the scope check entirely). + async fn allowed(db: &Pool, w_id: &str, tag: &str) -> bool { + let no_args = std::collections::HashMap::new(); + let args = windmill_queue::PushArgs::from(&no_args); + windmill_queue::check_tag_available_for_push(db, w_id, tag, &args, false, None) + .await + .is_ok() + } + // The ancestor chain is cached process-wide by workspace id, so use one no other test takes. let fork = "wm-fork-tagmarker"; sqlx::query!( @@ -5538,27 +5699,18 @@ async fn test_fork_marker_tag_admission_through_lineage(db: Pool) -> a "bare(test-workspace)".to_string(), ]))); - // A non-superadmin caller (a superadmin would bypass the scope check entirely). - let is_super_admin = false; - for (w_id, tag) in [("test-workspace", "bare"), ("test-workspace", "forky")] { assert!( - check_tag_available_for_workspace_internal(&db, w_id, tag, is_super_admin, None) - .await - .is_ok(), + allowed(&db, w_id, tag).await, "{tag} should be available in the workspace it names" ); } assert!( - check_tag_available_for_workspace_internal(&db, fork, "forky", is_super_admin, None) - .await - .is_ok(), + allowed(&db, fork, "forky").await, "a `*` tag must be granted to a fork through its parent lineage" ); assert!( - check_tag_available_for_workspace_internal(&db, fork, "bare", is_super_admin, None) - .await - .is_err(), + !allowed(&db, fork, "bare").await, "an unmarked tag must not reach a fork of the workspace it names" ); @@ -5567,6 +5719,122 @@ async fn test_fork_marker_tag_admission_through_lineage(db: Pool) -> a Ok(()) } +/// A workspace-scoped pattern admits the tags in its range only from the workspaces its scope +/// allows, and a tag listed by its own name with a scope stays inside that scope whatever pattern +/// it fits. The fork lineage and `$workspace` are resolved for a matching pattern. +#[sqlx::test(fixtures("base"))] +#[serial] +async fn test_scoped_custom_tag_pattern_admission(db: Pool) -> anyhow::Result<()> { + use windmill_common::worker::{CustomTags, CUSTOM_TAGS_PER_WORKSPACE}; + + initialize_tracing().await; + + // For a non-superadmin caller (a superadmin would bypass the scope check entirely). + async fn check(db: &Pool, w_id: &str, tag: &str, size: &str) -> Result<(), String> { + let args = std::collections::HashMap::from([( + "size".to_string(), + serde_json::value::to_raw_value(size).unwrap(), + )]); + let args = windmill_queue::PushArgs::from(&args); + windmill_queue::check_tag_available_for_push(db, w_id, tag, &args, false, None) + .await + .map_err(|e| e.to_string()) + } + async fn allowed(db: &Pool, w_id: &str, tag: &str, size: &str) -> bool { + check(db, w_id, tag, size).await.is_ok() + } + async fn allowed_as_written(db: &Pool, w_id: &str, tag: &str) -> bool { + windmill_common::jobs::check_tag_available_for_workspace_internal( + db, + w_id, + tag, + None, + std::future::ready(w_id.to_string()), + false, + None, + ) + .await + .is_ok() + } + + // The ancestor chain is cached process-wide by workspace id, so use one no other test takes. + let fork = "wm-fork-tagpattern"; + sqlx::query!( + "INSERT INTO workspace (id, name, owner, parent_workspace_id) + VALUES ($1, $1, 'test-user', 'test-workspace')", + fork + ) + .execute(&db) + .await?; + + CUSTOM_TAGS_PER_WORKSPACE.store(std::sync::Arc::new(CustomTags::from(vec![ + "gpu-$args[size]".to_string(), + "$args[size]-secret(test-workspace)".to_string(), + "gpu-secret(other)".to_string(), + "$workspace-$args[size](test-workspace*)".to_string(), + "urgent".to_string(), + "urgent(other)".to_string(), + ]))); + + let confined = check(&db, "test-workspace", "gpu-$args[size]", "secret").await; + let checks = [ + ( + "scoped pattern from its workspace", + allowed(&db, "test-workspace", "cpu-secret", "").await, + ), + ( + "scoped pattern refused elsewhere", + !allowed(&db, "other", "cpu-secret", "").await, + ), + ("confined tag refused past the patterns it fits", confined.is_err()), + ( + "global pattern for the tags nothing confines", + allowed(&db, "test-workspace", "gpu-large", "").await, + ), + ( + "confined tag from its workspace", + allowed(&db, "other", "gpu-secret", "").await, + ), + // Listing the tag by name without a scope as well keeps it open everywhere. + ( + "tag listed both globally and scoped", + allowed(&db, "test-workspace", "urgent", "").await, + ), + // The fork's `$workspace` is its parent's, whose tags its lineage reaches. Written as the + // resolved tag, not as the entry, so only the pattern can admit it. + ( + "fork-scoped `$workspace` pattern from a fork", + allowed(&db, fork, "test-workspace-large", "").await, + ), + ( + "fork-scoped `$workspace` pattern refused elsewhere", + !allowed(&db, "other", "$workspace-$args[size]", "large").await, + ), + ( + "as written from its workspace", + allowed_as_written(&db, "test-workspace", "$args[size]-secret").await, + ), + ( + "as written refused elsewhere", + !allowed_as_written(&db, "other", "$args[size]-secret").await, + ), + ]; + CUSTOM_TAGS_PER_WORKSPACE.store(std::sync::Arc::new(CustomTags::default())); + + for (what, ok) in checks { + assert!(ok, "{what}"); + } + let confined = confined.unwrap_err(); + assert!( + confined.contains("the custom tag gpu-secret(other) restricts it to other workspaces") + && confined.contains("gpu-$args[size]") + && confined.contains("$args[size]-secret(test-workspace)"), + "got {confined}" + ); + + Ok(()) +} + #[cfg(all(feature = "quickjs", feature = "python"))] #[sqlx::test(fixtures("base"))] async fn test_whileloop_propagates_inner_iterator_eval_failure( diff --git a/backend/windmill-ai/src/ai_types.rs b/backend/windmill-ai/src/ai_types.rs index 69c8ac4538..599cf0cca3 100644 --- a/backend/windmill-ai/src/ai_types.rs +++ b/backend/windmill-ai/src/ai_types.rs @@ -210,7 +210,9 @@ pub fn validate_model_pricing_json(ai_config: &serde_json::Value) -> Result<(), return Err(format!("Price override for {} is not an object", key)); }; for field in ["input", "output", "cache_read", "cache_write"] { - let Some(rate) = price.get(field) else { continue }; + let Some(rate) = price.get(field) else { + continue; + }; let rate = rate .as_f64() .filter(|r| r.is_finite() && *r >= 0.0 && *r <= MAX_MODEL_RATE); @@ -223,9 +225,98 @@ pub fn validate_model_pricing_json(ai_config: &serde_json::Value) -> Result<(), } for required in ["input", "output"] { if !price.contains_key(required) { - return Err(format!("Price override for {} is missing {}", key, required)); + return Err(format!( + "Price override for {} is missing {}", + key, required + )); } } } Ok(()) } + +// ============================================================================ +// Per-model token maps +// ============================================================================ + +/// One `provider:model` -> tokens map of the AI config, with the bounds the API +/// documents for it. Both maps are validated the same way, so the bounds live here +/// rather than at each call site. +pub struct TokenMap { + pub field: &'static str, + pub label: &'static str, + pub min: i64, + pub max: i64, +} + +/// The chat compacts its history as it nears the window, so a value below any real +/// model's window would compact every turn; the ceiling is far above any shipped +/// model and only catches typos. +pub const CONTEXT_WINDOWS: TokenMap = TokenMap { + field: "context_window_per_model", + label: "Context window", + min: 1024, + max: 10_000_000, +}; + +/// The completion cap the chat asks the provider for. +pub const OUTPUT_LIMITS: TokenMap = + TokenMap { field: "max_tokens_per_model", label: "Output limit", min: 1, max: 2_000_000 }; + +pub const TOKEN_MAPS: [TokenMap; 2] = [CONTEXT_WINDOWS, OUTPUT_LIMITS]; + +pub fn validate_token_limit(map: &TokenMap, key: &str, tokens: i64) -> Result<(), String> { + if (map.min..=map.max).contains(&tokens) { + Ok(()) + } else { + Err(format!( + "{} for {} must be between {} and {} tokens", + map.label, key, map.min, map.max + )) + } +} + +/// Bound these maps in the untyped instance AI config, for the reason given on +/// `validate_model_pricing_json`. +pub fn validate_token_maps_json(ai_config: &serde_json::Value) -> Result<(), String> { + for map in TOKEN_MAPS.iter() { + let entries = match ai_config.get(map.field) { + None | Some(serde_json::Value::Null) => continue, + Some(v) => v + .as_object() + .ok_or_else(|| format!("{} must be an object", map.field))?, + }; + for (key, tokens) in entries { + let tokens = tokens + .as_i64() + .ok_or_else(|| format!("{} for {} must be an integer", map.label, key))?; + validate_token_limit(map, key, tokens)?; + } + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + #[test] + fn instance_token_maps_are_bounded_integers() { + assert!(validate_token_maps_json(&json!({})).is_ok()); + for map in TOKEN_MAPS.iter() { + let entry = |value| json!({ map.field: { "customai:m": value } }); + assert!(validate_token_maps_json(&entry(json!(map.min))).is_ok()); + assert!(validate_token_maps_json(&entry(json!(map.max))).is_ok()); + for bad in [ + json!(map.min - 1), + json!(map.max + 1), + json!(4096.5), + json!("4096"), + ] { + assert!(validate_token_maps_json(&entry(bad)).is_err()); + } + assert!(validate_token_maps_json(&json!({ map.field: [4096] })).is_err()); + } + } +} diff --git a/backend/windmill-api-flows/src/flows.rs b/backend/windmill-api-flows/src/flows.rs index 5ac0e0588c..c7df69879e 100644 --- a/backend/windmill-api-flows/src/flows.rs +++ b/backend/windmill-api-flows/src/flows.rs @@ -51,7 +51,10 @@ use windmill_common::{ jobs::JobPayload, schedule::Schedule, triggers::MovedNativeTrigger, - utils::{http_get_from_hub, not_found_if_none, paginate, Pagination, RunnableKind, StripPath}, + utils::{ + http_get_from_hub, not_found_if_none, paginate, paginate_optional, Pagination, + RunnableKind, StripPath, + }, }; use windmill_dep_map::scoped_dependency_map::ScopedDependencyMap; use windmill_git_sync::{handle_deployment_metadata, DeployedObject}; @@ -747,6 +750,14 @@ async fn create_flow( .execute(&mut *tx) .await?; } + windmill_common::user_drafts::clear_draft_moves_from( + &mut tx, + &w_id, + &[UserDraftItemKind::Flow], + &nf.path, + None, + ) + .await?; audit_log( &mut *tx, @@ -915,25 +926,36 @@ pub struct FlowVersion { pub created_at: chrono::DateTime, #[serde(skip_serializing_if = "Option::is_none")] pub deployment_msg: Option, + /// Who deployed this version — the diff's version picker names them so a reader + /// can tell their own deploys from a teammate's. + #[serde(skip_serializing_if = "Option::is_none")] + pub created_by: Option, } async fn get_flow_history( authed: ApiAuthed, Extension(user_db): Extension, Path((w_id, path)): Path<(String, StripPath)>, + Query(pagination): Query, ) -> JsonResult> { let path = path.to_path(); check_scopes(&authed, || format!("flows:read:{}", path))?; + // Unasked-for, this listing stays whole: the history panels, the restart picker and + // the CLI all read it without paging. The diff picker asks for a page. + let (per_page, offset) = paginate_optional(pagination); let mut tx = user_db.begin(&authed).await?; let flows = sqlx::query_as!( FlowVersion, - "SELECT flow_version.id, flow_version.created_at, deployment_metadata.deployment_msg FROM flow_version + "SELECT flow_version.id, flow_version.created_at, flow_version.created_by, deployment_metadata.deployment_msg FROM flow_version LEFT JOIN deployment_metadata ON flow_version.id = deployment_metadata.flow_version - WHERE flow_version.path = $1 AND flow_version.workspace_id = $2 - ORDER BY flow_version.created_at DESC", + WHERE flow_version.path = $1 AND flow_version.workspace_id = $2 + ORDER BY flow_version.created_at DESC + LIMIT $3 OFFSET $4", path, - w_id + w_id, + per_page, + offset, ) .fetch_all(&mut *tx) .await?; @@ -953,7 +975,7 @@ async fn get_latest_version( let version = sqlx::query_as!( FlowVersion, - "SELECT flow_version.id, flow_version.created_at, deployment_metadata.deployment_msg FROM flow_version + "SELECT flow_version.id, flow_version.created_at, flow_version.created_by, deployment_metadata.deployment_msg FROM flow_version LEFT JOIN deployment_metadata ON flow_version.id = deployment_metadata.flow_version WHERE flow_version.path = $1 AND flow_version.workspace_id = $2 ORDER BY flow_version.created_at DESC", @@ -1438,6 +1460,20 @@ async fn update_flow( .await?; } + if is_new_path { + // Everything left at the old path is a draft this deploy didn't consume + // — teammates' rows, and the deployer's own when the caller asked us to + // keep it. Carry them rather than strand them. + windmill_common::user_drafts::move_drafts_for_path( + &mut tx, + &w_id, + &[UserDraftItemKind::Flow], + flow_path, + &nf.path, + ) + .await?; + } + audit_log( &mut *tx, &authed, diff --git a/backend/windmill-api-integration-tests/tests/datatable_acl.rs b/backend/windmill-api-integration-tests/tests/datatable_acl.rs new file mode 100644 index 0000000000..4310830ae2 --- /dev/null +++ b/backend/windmill-api-integration-tests/tests/datatable_acl.rs @@ -0,0 +1,108 @@ +//! Who may read and change a data table's grants and owners. On the Enterprise Edition: its +//! administrators, from the workspace that governs it. Without it: nobody. Each refusal is decided +//! before anything connects to the data table, so the fixture's database never has to exist. + +use serde_json::{json, Value}; +use sqlx::{Pool, Postgres}; + +use windmill_test_utils::*; + +fn grant_select_on_public() -> Value { + json!({ + "target": {"kind": "schema", "schema": "public"}, + "change": {"type": "grant", "role": "analytics", "privileges": ["SELECT"], + "scope": "all_tables"}, + "statements": [r#"GRANT SELECT ON ALL TABLES IN SCHEMA "public" TO "analytics""#] + }) +} + +async fn post_acl( + port: u16, + w_id: &str, + action: &str, + token: &str, +) -> anyhow::Result { + Ok(reqwest::Client::new() + .post(format!( + "http://localhost:{port}/api/w/{w_id}/workspaces/datatable_acl/main/{action}" + )) + .header("Authorization", format!("Bearer {token}")) + .json(&grant_select_on_public()) + .send() + .await?) +} + +/// A fork reaches the data table through a pointer: it may use it, never change what each role may +/// touch on it — not even as an admin of the fork. +#[cfg(all(feature = "private", feature = "enterprise"))] +#[sqlx::test(migrations = "../migrations", fixtures("base", "datatable_roles"))] +async fn a_fork_cannot_change_access_on_the_data_table_it_points_at( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + for action in ["plan", "apply"] { + let resp = post_acl(port, "wm-fork-dt", action, "SECRET_TOKEN_2").await?; + assert_eq!(resp.status(), 401, "{action}: {}", resp.text().await?); + } + Ok(()) +} + +#[cfg(all(feature = "private", feature = "enterprise"))] +#[sqlx::test(migrations = "../migrations", fixtures("base", "datatable_roles"))] +async fn a_member_who_is_not_an_admin_cannot_change_access( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + for action in ["plan", "apply"] { + let resp = post_acl(port, "test-workspace", action, "SECRET_TOKEN_2").await?; + assert_eq!(resp.status(), 401, "{action}: {}", resp.text().await?); + } + Ok(()) +} + +/// Not even reading, and not even on a data table that is not under roles — which any member +/// reaches, so only the edition stands between them and the instance's credentials. +#[cfg(not(all(feature = "private", feature = "enterprise")))] +#[sqlx::test(migrations = "../migrations", fixtures("base", "datatable_roles"))] +async fn only_the_enterprise_edition_has_the_access_editor( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + sqlx::query( + "UPDATE workspace_settings + SET datatable = datatable #- '{datatables,main,permissions}' + WHERE workspace_id = 'test-workspace'", + ) + .execute(&db) + .await?; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + let read = reqwest::Client::new() + .get(format!( + "http://localhost:{port}/api/w/test-workspace/workspaces/datatable_acl/main?kind=database" + )) + .header("Authorization", "Bearer SECRET_TOKEN") + .send() + .await?; + let mut responses = vec![("read", read)]; + for action in ["plan", "apply"] { + responses.push(( + action, + post_acl(port, "test-workspace", action, "SECRET_TOKEN").await?, + )); + } + for (action, resp) in responses { + assert_eq!(resp.status(), 400, "{action}"); + let body = resp.text().await?; + assert!( + body.contains("Data table roles are a Windmill Enterprise Edition feature"), + "{action}: {body}" + ); + } + Ok(()) +} diff --git a/backend/windmill-api-integration-tests/tests/datatable_roles.rs b/backend/windmill-api-integration-tests/tests/datatable_roles.rs index 57b45088bd..4948a0098a 100644 --- a/backend/windmill-api-integration-tests/tests/datatable_roles.rs +++ b/backend/windmill-api-integration-tests/tests/datatable_roles.rs @@ -627,21 +627,234 @@ async fn a_rename_has_to_match_the_save_it_claims_to_describe( Ok(()) } +async fn database_exists(db: &Pool, name: &str) -> anyhow::Result { + Ok( + sqlx::query_scalar("SELECT EXISTS (SELECT 1 FROM pg_database WHERE datname = $1)") + .bind(name) + .fetch_one(db) + .await?, + ) +} + +async fn workspace_exists(db: &Pool, id: &str) -> anyhow::Result { + Ok( + sqlx::query_scalar("SELECT EXISTS (SELECT 1 FROM workspace WHERE id = $1)") + .bind(id) + .fetch_one(db) + .await?, + ) +} + #[sqlx::test(migrations = "../migrations", fixtures("base", "datatable_roles"))] -async fn a_data_table_under_roles_is_not_copied_into_a_fork( +async fn a_data_table_under_roles_is_cloned_only_with_its_grants( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + // The database a fork's copy of `main` goes into is named after the fork. Every request below + // is refused before it is created, so the cluster-wide name never collides with a sibling run. + let target = "wm_fork_copy__main"; + let fork = |token: &str, forked: Value| { + authed( + client().post(format!( + "http://localhost:{port}/api/w/test-workspace/workspaces/create_fork" + )), + token, + ) + .json(&json!({"id": "wm-fork-copy", "name": "copy", "forked_datatables": [forked]})) + }; + + // Rows are a workspace admin's to copy, as they were before roles. + let resp = fork( + "SECRET_TOKEN_2", + json!({"name": "main", "new_dbname": target, "fork_behavior": "schema_and_data"}), + ) + .send() + .await?; + assert_eq!(resp.status(), 403, "{}", resp.text().await?); + assert!(!database_exists(&db, target).await?); + + // Even the schema alone lists every table, which a member covered by no role cannot read in + // the parent. + let resp = fork( + "SECRET_TOKEN_3", + json!({"name": "main", "new_dbname": target, "fork_behavior": "schema_only"}), + ) + .send() + .await?; + assert_eq!(resp.status(), 401, "{}", resp.text().await?); + assert!(!database_exists(&db, target).await?); + + // Refused in the first phase too, before a git branch is created for a fork that cannot be. + let resp = authed( + client().post(format!( + "http://localhost:{port}/api/w/test-workspace/workspaces/create_workspace_fork_branch" + )), + "SECRET_TOKEN_3", + ) + .json( + &json!({"id": "wm-fork-copy", "name": "copy", "forked_datatables": [ + {"name": "main", "new_dbname": target, "fork_behavior": "schema_only"} + ]}), + ) + .send() + .await?; + assert_eq!(resp.status(), 401, "{}", resp.text().await?); + + // A database the request did not create — whatever the data table, under roles or not — may be + // a copy left behind by a deleted fork, which the entry would reach without its governance. + let resp = fork( + "SECRET_TOKEN", + json!({"name": "other", "new_dbname": "wm_fork_copy__other"}), + ) + .send() + .await?; + assert_eq!(resp.status(), 400); + assert!( + resp.text().await?.contains("fork_behavior"), + "a database the fork did not create was taken" + ); + assert!(!workspace_exists(&db, "wm-fork-copy").await?); + + // The copy an admin asks for goes ahead in an edition that replays grants, and is refused + // before any database exists in one that does not. The fixture's database does not exist, so + // the dump fails either way — and leaves neither a database nor a fork behind. + let resp = fork( + "SECRET_TOKEN", + json!({"name": "main", "new_dbname": target, "fork_behavior": "schema_only"}), + ) + .send() + .await?; + assert_ne!(resp.status(), 200); + let body = resp.text().await?; + #[cfg(all(feature = "private", feature = "enterprise"))] + assert!(body.contains("pg_dump"), "{body}"); + #[cfg(not(all(feature = "private", feature = "enterprise")))] + assert!(body.contains("Enterprise Edition"), "{body}"); + assert!(!database_exists(&db, target).await?); + assert!(!workspace_exists(&db, "wm-fork-copy").await?); + Ok(()) +} + +#[sqlx::test(migrations = "../migrations", fixtures("base", "datatable_roles"))] +async fn a_clone_takes_its_roles_from_the_data_table_it_was_cloned_from( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + sqlx::query( + r#"UPDATE workspace_settings SET datatable = jsonb_set(datatable, '{datatables,copy}', '{ + "database": {"resource_type": "instance", "resource_path": "wm_fork_dt__copy"}, + "governed_by": {"workspace_id": "test-workspace", "datatable": "main"}, + "forked_from": {} + }'::jsonb) WHERE workspace_id = 'wm-fork-dt'"#, + ) + .execute(&db) + .await?; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let fork = format!("http://localhost:{port}/api/w/wm-fork-dt/workspaces"); + + // `test-user-2` administers the fork, and is only a tenant of `analytics` in the parent. + let resp = authed( + client().get(format!("{fork}/datatable_usable_roles/copy")), + "SECRET_TOKEN_2", + ) + .send() + .await?; + let body: Value = resp.json().await?; + assert_eq!(body["roles"], json!(["analytics"]), "{body}"); + + let resp = authed( + client().get(format!("{fork}/datatable_permissions/copy")), + "SECRET_TOKEN_2", + ) + .send() + .await?; + let body: Value = resp.json().await?; + assert_eq!(body["editable"], false, "{body}"); + assert_eq!(body["clone_of"]["workspace_id"], "test-workspace", "{body}"); + + // Nobody changes a clone's roles, a superadmin included: they are the source's. + for token in ["SECRET_TOKEN_2", "SECRET_TOKEN"] { + let resp = authed( + client().post(format!("{fork}/datatable_permissions/copy")), + token, + ) + .json(&json!({"permissioned": false})) + .send() + .await?; + assert_eq!(resp.status(), 400, "{token} changed a clone's roles"); + } + + // A settings save cannot clear the link. + let resp = authed( + client().post(format!("{fork}/edit_datatable_config")), + "SECRET_TOKEN_2", + ) + .json(&json!({ + "settings": {"datatables": {"main": {}, "copy": { + "database": {"resource_type": "instance", "resource_path": "wm_fork_dt__copy"} + }}}, + "renames": [], "deleted_datatables": [] + })) + .send() + .await?; + assert_eq!(resp.status(), 200, "{}", resp.text().await?); + let governed_by: Option = sqlx::query_scalar( + "SELECT datatable->'datatables'->'copy'->'governed_by' FROM workspace_settings + WHERE workspace_id = 'wm-fork-dt'", + ) + .fetch_one(&db) + .await?; + assert_eq!(governed_by.unwrap()["datatable"], "main"); + + // Nor move it, a superadmin included: its grants were replayed into that database alone. + let resp = authed( + client().post(format!("{fork}/edit_datatable_config")), + "SECRET_TOKEN", + ) + .json(&json!({ + "settings": {"datatables": {"main": {}, "copy": { + "database": {"resource_type": "instance", "resource_path": "wm_fork_dt__elsewhere"} + }}}, + "renames": [], "deleted_datatables": [] + })) + .send() + .await?; + assert_eq!(resp.status(), 400, "{}", resp.text().await?); + + // Nor reach its database through a second entry without roles, which would connect as `admin`. + let resp = authed( + client().post(format!( + "http://localhost:{port}/api/w/test-workspace/workspaces/edit_datatable_config" + )), + "SECRET_TOKEN", + ) + .json(&json!({ + "settings": {"datatables": { + "main": {"database": {"resource_type": "instance", "resource_path": "dt_main"}}, + "alias": {"database": {"resource_type": "instance", "resource_path": "wm_fork_dt__copy"}} + }}, + "renames": [], "deleted_datatables": [] + })) + .send() + .await?; + let status = resp.status(); + let body = resp.text().await?; + assert_eq!(status, 400, "{body}"); + assert!(body.contains("without carrying those roles"), "{body}"); + Ok(()) +} + +#[sqlx::test(migrations = "../migrations", fixtures("base", "datatable_roles"))] +async fn a_data_table_under_roles_is_not_copied_without_its_grants( db: Pool, ) -> anyhow::Result<()> { initialize_tracing().await; - // `pg_dump` carries no roles and the restore drops ACLs, so a copy would arrive with the - // parent's tenants and none of the grants behind them: every role but admin denied by - // Postgres in a data table that reads as configured. Refuse the copy rather than ship that, - // and refuse it before any data moves. let server = ApiServer::start(db.clone()).await?; let port = server.addr.port(); - // Both halves of the clone: the database the copy would land in, then the copy itself. The - // first has to refuse too, or a permissioned fork leaves an empty registered database that - // no data table entry names and nothing collects. let resp = authed( client().post(format!( "http://localhost:{port}/api/w/test-workspace/workspaces/create_pg_database" @@ -912,6 +1125,18 @@ async fn a_stored_name_containing_a_question_mark_resolves_as_itself( resolve("main?dt").await.is_err(), "an unknown parameter was ignored" ); + + sqlx::query( + "UPDATE workspace_settings + SET datatable = jsonb_set(datatable, '{datatables,main?role=analytics}', datatable->'datatables'->'main') + WHERE workspace_id = 'test-workspace'", + ) + .execute(&db) + .await?; + assert!( + resolve("main?role=analytics").await.is_err(), + "a reference naming both a stored data table and a role on another resolved to one of them" + ); Ok(()) } @@ -1009,6 +1234,70 @@ async fn an_entry_without_roles_cannot_newly_reach_a_database_under_roles( Ok(()) } +/// Browsing names the role it connects as, and a role the caller may not use is refused rather +/// than quietly listed as the default. The refusal is decided before connecting, so the fixture's +/// database never has to exist. +#[cfg(all(feature = "private", feature = "enterprise"))] +#[sqlx::test(migrations = "../migrations", fixtures("base", "datatable_roles"))] +async fn browsing_as_a_role_the_caller_may_not_use_is_refused( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let base = format!("http://localhost:{port}/api/w/test-workspace/workspaces"); + + // `test-user-2` is a tenant of `analytics` only. + let resp = authed( + client().get(format!( + "{base}/list_datatable_tables?role_for=main&role=admin" + )), + "SECRET_TOKEN_2", + ) + .send() + .await?; + assert_eq!(resp.status(), 200); + let body: Value = resp.json().await?; + let entry = body + .as_array() + .and_then(|a| a.iter().find(|e| e["datatable_name"] == "main")) + .expect("main is listed"); + assert_eq!(entry["usable_roles"], json!(["analytics"]), "{entry}"); + assert_eq!(entry["default_role"], "analytics", "{entry}"); + assert_eq!(entry["permissioned"], true, "{entry}"); + assert_eq!(entry["instance"], true, "{entry}"); + let error = entry["error"].as_str().unwrap_or_default(); + assert!( + error.contains("Not allowed to use role 'admin'"), + "listed as another role than the one asked for: {entry}" + ); + + let resp = authed( + client().get(format!( + "{base}/get_datatable_table_schema?datatable_name=main&schema_name=public&table_name=t&role=admin" + )), + "SECRET_TOKEN_2", + ) + .send() + .await?; + let status = resp.status(); + let text = resp.text().await?; + assert!( + text.contains("Not allowed to use role 'admin'"), + "{status}: {text}" + ); + + // A role means nothing without the data table it belongs to. + let resp = authed( + client().get(format!("{base}/list_datatable_tables?role=analytics")), + "SECRET_TOKEN_2", + ) + .send() + .await?; + assert_eq!(resp.status(), 400, "{}", resp.text().await?); + Ok(()) +} + #[sqlx::test(migrations = "../migrations", fixtures("base", "datatable_roles"))] async fn an_alias_saved_elsewhere_waits_for_roles_going_on_for_its_database( db: Pool, @@ -1125,6 +1414,158 @@ async fn a_fork_waits_for_a_rename_of_the_data_table_it_keeps( Ok(()) } +async fn enable_migrations_on_main(db: &Pool) -> anyhow::Result<()> { + sqlx::query( + "UPDATE workspace_settings + SET datatable = jsonb_set(datatable, '{datatables,main,migrations_enabled}', 'true') + WHERE workspace_id = 'test-workspace'", + ) + .execute(db) + .await?; + Ok(()) +} + +#[cfg(all(feature = "private", feature = "enterprise"))] +#[sqlx::test(migrations = "../migrations", fixtures("base", "datatable_roles"))] +async fn a_migration_is_written_only_by_who_may_run_it(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + enable_migrations_on_main(&db).await?; + let server = ApiServer::start(db.clone()).await?; + let base = format!( + "http://localhost:{}/api/w/test-workspace/workspaces", + server.addr.port() + ); + let create = |token: &str, body: Value| { + authed( + client() + .post(format!("{base}/create_datatable_migration/main")) + .json(&body), + token, + ) + .send() + }; + let upsert = |token: &str, code_up: &str| { + authed( + client() + .post(format!("{base}/upsert_datatable_migration/main")) + .json(&json!({"timestamp": 20260101000000i64, "name": "m", "code_up": code_up})), + token, + ) + .send() + }; + + // `test-user-2` holds `analytics` and nothing else, so of a migration they may store only + // what they could run: SQL that declares no role runs as admin, in the up or the down. + for body in [ + json!({"name": "m", "code_up": "DROP TABLE t"}), + json!({"name": "m", "code_up": "-- role analytics\nSELECT 1", "code_down": "DROP TABLE t"}), + ] { + let resp = create("SECRET_TOKEN_2", body.clone()).await?; + assert_eq!(resp.status(), 401, "{body}: {}", resp.text().await?); + } + let resp = create( + "SECRET_TOKEN_2", + json!({"name": "m", "code_up": "-- role analytics\nSELECT 1"}), + ) + .await?; + assert_eq!(resp.status(), 200, "{}", resp.text().await?); + + // An admin's migration re-pushed as stored writes nothing, so it passes. Replacing it removes + // SQL they could not run, even with SQL they could, so it is refused like deleting it. + let resp = upsert("SECRET_TOKEN", "DROP TABLE t").await?; + assert_eq!(resp.status(), 200, "{}", resp.text().await?); + let resp = upsert("SECRET_TOKEN_2", "DROP TABLE t").await?; + assert_eq!(resp.status(), 200, "{}", resp.text().await?); + for code_up in ["DROP TABLE u", "-- role analytics\nSELECT 1"] { + let resp = upsert("SECRET_TOKEN_2", code_up).await?; + assert_eq!(resp.status(), 401, "{code_up}: {}", resp.text().await?); + } + let resp = authed( + client().delete(format!( + "{base}/delete_datatable_migration/main/20260101000000" + )), + "SECRET_TOKEN_2", + ) + .send() + .await?; + assert_eq!(resp.status(), 401, "{}", resp.text().await?); + + let code_up: String = sqlx::query_scalar( + "SELECT code_up FROM datatable_migrations + WHERE workspace_id = 'test-workspace' AND timestamp = 20260101000000", + ) + .fetch_one(&db) + .await?; + assert_eq!(code_up, "DROP TABLE t"); + Ok(()) +} + +#[sqlx::test(migrations = "../migrations", fixtures("base", "datatable_roles"))] +async fn an_operator_neither_writes_nor_runs_a_migration(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + enable_migrations_on_main(&db).await?; + // Off roles, where every role check passes everyone, so only the operator guard is left to + // refuse. + sqlx::query( + "UPDATE workspace_settings SET datatable = datatable #- '{datatables,main,permissions}' + WHERE workspace_id = 'test-workspace'", + ) + .execute(&db) + .await?; + // A token of its own: the auth cache is process-wide, and the other tests here authenticate + // `SECRET_TOKEN_3` in this workspace as a non-operator. + sqlx::raw_sql( + "UPDATE usr SET operator = true, role = 'Operator' + WHERE workspace_id = 'test-workspace' AND username = 'test-user-3'; + INSERT INTO token (token_hash, token_prefix, token, email, label, super_admin) + VALUES (encode(sha256('DT_OPERATOR_TOKEN'::bytea), 'hex'), 'DT_OPERATO', + 'DT_OPERATOR_TOKEN', 'test3@windmill.dev', 'operator', false);", + ) + .execute(&db) + .await?; + let server = ApiServer::start(db.clone()).await?; + let base = format!( + "http://localhost:{}/api/w/test-workspace/workspaces", + server.addr.port() + ); + let migration = json!({"timestamp": 20260101000000i64, "name": "m", "code_up": "SELECT 1"}); + + for request in [ + client() + .post(format!("{base}/create_datatable_migration/main")) + .json(&migration), + client() + .post(format!("{base}/upsert_datatable_migration/main")) + .json(&migration), + client().delete(format!( + "{base}/delete_datatable_migration/main/20260101000000" + )), + client().post(format!("{base}/run_datatable_migrations/main")), + client().post(format!("{base}/rollback_datatable_migrations/main")), + client().post(format!("{base}/generate_initial_datatable_migration/main")), + ] { + let resp = authed(request, "DT_OPERATOR_TOKEN").send().await?; + let status = resp.status(); + let text = resp.text().await?; + assert!( + status == 401 && text.contains("Operators cannot"), + "{status}: {text}" + ); + } + + // The same write from a developer is accepted: what refused the operator is theirs alone. + let resp = authed( + client() + .post(format!("{base}/upsert_datatable_migration/main")) + .json(&migration), + "SECRET_TOKEN_2", + ) + .send() + .await?; + assert_eq!(resp.status(), 200, "{}", resp.text().await?); + Ok(()) +} + #[cfg(not(all(feature = "private", feature = "enterprise")))] const ENTERPRISE_REFUSAL: &str = "Data table roles are a Windmill Enterprise Edition feature"; @@ -1245,3 +1686,73 @@ async fn without_the_enterprise_edition_a_data_table_under_roles_is_refused_a_co assert!(err.to_string().contains(ENTERPRISE_REFUSAL), "{err}"); Ok(()) } + +/// A save blocked on an instance database's lock is not a user of it until it commits one: the +/// cleanup for a database whose setup failed lets that save through and reads what it left. +#[sqlx::test(migrations = "../migrations", fixtures("base", "datatable_roles"))] +async fn cleanup_waits_out_a_save_racing_it_for_an_instance_database( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + let key = "instance_database:dt_probe"; + + // Queues cleanup behind `holder`, then a save behind cleanup, so cleanup takes the lock with + // the save already waiting on it — the ordering the waiter check is for. + let race = |commit: bool| { + let db = db.clone(); + async move { + let mut holder = db.acquire().await?; + sqlx::query("SELECT pg_advisory_lock(hashtext($1))") + .bind(key) + .execute(&mut *holder) + .await?; + + let cleanup = tokio::spawn({ + let db = db.clone(); + async move { windmill_common::drop_unused_instance_database(&db, "dt_probe").await } + }); + tokio::time::sleep(std::time::Duration::from_millis(300)).await; + + let save = tokio::spawn({ + let db = db.clone(); + async move { + let mut tx = db.begin().await?; + windmill_common::lock_instance_databases(&mut tx, ["dt_probe"]).await?; + sqlx::query( + r#"UPDATE workspace_settings SET datatable = jsonb_set(datatable, + '{datatables,probe}', + '{"database": {"resource_type": "instance", "resource_path": "dt_probe"}}') + WHERE workspace_id = 'test-workspace'"#, + ) + .execute(&mut *tx) + .await?; + if commit { + tx.commit().await?; + } else { + tx.rollback().await?; + } + Ok::<_, anyhow::Error>(()) + } + }); + tokio::time::sleep(std::time::Duration::from_millis(300)).await; + + sqlx::query("SELECT pg_advisory_unlock(hashtext($1))") + .bind(key) + .execute(&mut *holder) + .await?; + save.await??; + Ok::<_, anyhow::Error>(cleanup.await??) + } + }; + + assert!( + matches!(race(false).await?, windmill_common::Cleanup::Dropped), + "a save that rolled back kept the database, whose name then blocks every retry" + ); + let kept = race(true).await?; + let windmill_common::Cleanup::InUse(users) = kept else { + anyhow::bail!("the database was dropped under a save that committed a reference to it") + }; + assert_eq!(users, vec!["test-workspace".to_string()]); + Ok(()) +} diff --git a/backend/windmill-api-integration-tests/tests/fork_clone_on_behalf_of.rs b/backend/windmill-api-integration-tests/tests/fork_clone_on_behalf_of.rs index a886d26e17..bc6c10f1ee 100644 --- a/backend/windmill-api-integration-tests/tests/fork_clone_on_behalf_of.rs +++ b/backend/windmill-api-integration-tests/tests/fork_clone_on_behalf_of.rs @@ -407,9 +407,18 @@ async fn test_fork_repoints_unresolvable_identities(db: Pool) -> anyho text("SELECT value->'policy'->>'on_behalf_of' FROM draft WHERE workspace_id = 'wm-fork-repoint' AND path = 'u/test-user/stranger' AND typ = 'raw_app'").await?, "u/test-user" ); + // `clone_drafts` strips a NUL escape as it copies, so the row reaches the fork + // parseable and the repoint below reaches it like any other draft's. The rule this + // guards is that the fork completes and no identity naming nobody survives it; the + // skip only ever existed because `to_jsonb` raises on a value still holding one. assert_eq!( text("SELECT CASE WHEN strpos(value::text, 'u/test-user-2') > 0 THEN 'kept' ELSE 'rewritten' END FROM draft WHERE workspace_id = 'wm-fork-repoint' AND path = 'u/test-user/nul'").await?, - "kept" + "rewritten" + ); + // And it arrives without the poison that made it a special case. + assert_eq!( + text("SELECT CASE WHEN position(chr(92) || 'u0000' in value::text) > 0 THEN 'poisoned' ELSE 'clean' END FROM draft WHERE workspace_id = 'wm-fork-repoint' AND path = 'u/test-user/nul'").await?, + "clean" ); assert_eq!( text("SELECT value->>'permissioned_as' FROM draft WHERE workspace_id = 'wm-fork-repoint' AND typ = 'trigger_websocket'").await?, diff --git a/backend/windmill-api-integration-tests/tests/login_link.rs b/backend/windmill-api-integration-tests/tests/login_link.rs index 3906232075..5c3a231a48 100644 --- a/backend/windmill-api-integration-tests/tests/login_link.rs +++ b/backend/windmill-api-integration-tests/tests/login_link.rs @@ -115,60 +115,6 @@ async fn login_link_is_single_use_and_same_origin(db: Pool) -> anyhow: Ok(()) } -#[sqlx::test(migrations = "../migrations", fixtures("base"))] -async fn confirmed_login_link_is_spent_by_the_click_not_the_page( - db: Pool, -) -> anyhow::Result<()> { - initialize_tracing().await; - let server = ApiServer::start(db.clone()).await?; - let port = server.addr.port(); - let base = format!("http://localhost:{port}/api"); - - let resp = client() - .post(format!("{base}/users/login_links")) - .header("Authorization", "Bearer SECRET_TOKEN") - .json(&json!({"email": "test2@windmill.dev", "confirm": true})) - .send() - .await?; - assert_eq!(resp.status(), 201); - let link = resp.json::().await?; - // The URL handed out is the frontend page, not the API path that signs in on a GET. - let token = link["url"] - .as_str() - .unwrap() - .split_once("/user/login_link?token=") - .expect("confirmation page url") - .1 - .to_string(); - - let confirm = || { - client() - .post(format!("{base}/auth/login_link/{token}")) - .send() - }; - let resp = confirm().await?; - assert_eq!(resp.status(), 200); - assert!(resp - .headers() - .get_all("set-cookie") - .iter() - .any(|c| c.to_str().unwrap().starts_with("token="))); - assert_eq!( - resp.json::().await?["location"], - "/user/workspaces" - ); - - let resp = confirm().await?; - assert_eq!(resp.status(), 200); - assert!(resp.headers().get("set-cookie").is_none()); - assert_eq!( - resp.json::().await?["location"], - "/user/login_link_expired?reason=used" - ); - - Ok(()) -} - #[sqlx::test(migrations = "../migrations", fixtures("base"))] async fn login_link_mint_can_require_a_login_type(db: Pool) -> anyhow::Result<()> { initialize_tracing().await; diff --git a/backend/windmill-api-integration-tests/tests/native_triggers.rs b/backend/windmill-api-integration-tests/tests/native_triggers.rs index 059b3aa2d2..e7f0147323 100644 --- a/backend/windmill-api-integration-tests/tests/native_triggers.rs +++ b/backend/windmill-api-integration-tests/tests/native_triggers.rs @@ -12,7 +12,7 @@ use serde_json::json; use sqlx::{Pool, Postgres}; use axum::http::StatusCode; -use windmill_api_auth::ApiAuthed; +use windmill_api_auth::{ApiAuthed, ScopePathFilter}; use windmill_common::{ error::Error, variables::{build_crypt, encrypt}, @@ -661,6 +661,7 @@ async fn test_rename_moves_native_trigger(db: Pool) -> anyhow::Result< None, Some("f/test/after"), Some(false), + ScopePathFilter::AllowAll, ) .await?; assert_eq!( @@ -672,6 +673,55 @@ async fn test_rename_moves_native_trigger(db: Pool) -> anyhow::Result< Ok(()) } +// ============================================================================ +// 6. Token scope filtering on list +// ============================================================================ + +/// A `native_triggers:read:f/test/*` token must only list triggers under `f/test/`, on the `/` +/// boundary (`f/testx/` is out of scope). +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn test_list_native_triggers_scope_filter(db: Pool) -> anyhow::Result<()> { + for (i, path) in ["f/testx/out", "f/test/in"].into_iter().enumerate() { + insert_test_script(&db, path).await?; + store_native_trigger( + &db, + "test-workspace", + ServiceName::Nextcloud, + &format!("ext-{i}"), + &NativeTriggerConfig { + script_path: path.to_string(), + is_flow: false, + webhook_token: format!("abcdefghij123456789{i}"), + }, + json!({"event": "OCP\\Files\\Events\\Node\\NodeCreatedEvent"}), + None, + true, + ) + .await?; + } + + let triggers = list_native_triggers( + &db, + "test-workspace", + ServiceName::Nextcloud, + None, + None, + None, + None, + ScopePathFilter::Restricted { exact: vec![], prefix: vec!["f/test".to_string()] }, + ) + .await?; + assert_eq!( + triggers + .iter() + .map(|t| t.script_path.as_str()) + .collect::>(), + vec!["f/test/in"] + ); + + Ok(()) +} + // --- parse_stop_channel_params --- #[test] diff --git a/backend/windmill-api-integration-tests/tests/remote_deploy.rs b/backend/windmill-api-integration-tests/tests/remote_deploy.rs new file mode 100644 index 0000000000..27447c3095 --- /dev/null +++ b/backend/windmill-api-integration-tests/tests/remote_deploy.rs @@ -0,0 +1,297 @@ +use serde_json::json; +use sqlx::{Pool, Postgres}; + +use windmill_common::variables::{build_crypt, encrypt}; +use windmill_test_utils::*; + +fn client() -> reqwest::Client { + reqwest::Client::new() +} + +fn authed(builder: reqwest::RequestBuilder) -> reqwest::RequestBuilder { + builder.header("Authorization", "Bearer SECRET_TOKEN") +} + +/// The deploy target is this very server, which the proxy has no way to tell from another +/// instance: it only ever knows the configured URL, the caller's stored token, and the path. +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn test_remote_deploy_proxy(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let base = format!("http://localhost:{port}/api/w/test-workspace/remote_deploy"); + + // Planted rather than set through the route, which is enterprise-gated. Unlike the route, it + // deletes no token for another target, which is what a connect landing just after the + // route's cleanup leaves behind. + let set_target = |base_url: String, workspace_id: &'static str| { + let db = db.clone(); + async move { + sqlx::query!( + "UPDATE workspace_settings + SET remote_deploy_target = $1, remote_deploy_target_changed_at = now() + WHERE workspace_id = 'test-workspace'", + json!({ "base_url": base_url, "workspace_id": workspace_id }) + ) + .execute(&db) + .await + .unwrap(); + } + }; + set_target(format!("http://localhost:{port}"), "test-workspace").await; + + let resp = authed(client().get(format!("{base}/target"))) + .send() + .await?; + assert_eq!(resp.status(), 200); + let status = resp.json::().await?; + assert_eq!(status["target"]["workspace_id"], "test-workspace"); + assert!(status["connection"].is_null()); + + // Deploying before connecting names the step that is missing, rather than reaching the target + // with the caller's credentials for this instance. + let resp = authed(client().get(format!("{base}/proxy/none/users/whoami"))) + .send() + .await?; + assert_eq!(resp.status(), 400); + assert!(resp.text().await?.contains("Connect to")); + + let target = + json!({ "base_url": format!("http://localhost:{port}"), "workspace_id": "test-workspace" }); + + // A token obtained for another target is refused before it is sent anywhere. + let resp = authed(client().post(format!("{base}/connect"))) + .json(&json!({"token": "SECRET_TOKEN", "target": { "base_url": format!("http://localhost:{port}"), "workspace_id": "elsewhere" }})) + .send() + .await?; + assert_eq!(resp.status(), 400); + + // A token the target refuses is not stored. + let resp = authed(client().post(format!("{base}/connect"))) + .json(&json!({"token": "not-a-token", "target": target})) + .send() + .await?; + assert_eq!(resp.status(), 400); + + let resp = authed(client().post(format!("{base}/connect"))) + .json(&json!({"token": "SECRET_TOKEN", "target": target})) + .send() + .await?; + assert_eq!(resp.status(), 200); + let connection = resp.json::().await?; + assert_eq!(connection["remote_email"], "test@windmill.dev"); + let key = connection["proxy_key"].as_str().unwrap().to_string(); + + let resp = authed(client().get(format!("{base}/proxy/{key}/users/whoami"))) + .send() + .await?; + assert_eq!(resp.status(), 200); + // Whatever the remote returns is served from this origin, so it must never render as a page. + let csp = resp.headers()["content-security-policy"] + .to_str()? + .to_string(); + assert!(csp.starts_with("sandbox"), "{csp}"); + assert_eq!( + resp.json::().await?["email"], + "test@windmill.dev" + ); + + // A link from elsewhere rides the session cookie but cannot know the key, so it cannot spend + // the stored token. + // Nor can a credential that may not use the proxy read the key, to build such a link itself. + sqlx::query!( + "INSERT INTO token (token_hash, token_prefix, token, email, label, super_admin, read_only) + VALUES (encode(sha256('READ_ONLY_TOKEN'::bytea), 'hex'), 'READ_ONLY_', 'READ_ONLY_TOKEN', + 'test@windmill.dev', 'read only', false, true)" + ) + .execute(&db) + .await?; + let resp = client() + .get(format!("{base}/target")) + .header("Authorization", "Bearer READ_ONLY_TOKEN") + .send() + .await?; + assert_eq!(resp.status(), 200); + assert!(resp.json::().await?["connection"].is_null()); + + let guessed = "x".repeat(key.len()); + let resp = authed(client().get(format!("{base}/proxy/{guessed}/users/whoami"))) + .send() + .await?; + assert_eq!(resp.status(), 400); + + // A connect locks nothing against a key rotation, so its row can land under the old key: that + // is no connection, which the drawer offers to replace, rather than an error on every deploy. + sqlx::query!( + "UPDATE remote_deploy_token SET token = 'not-under-this-key' + WHERE workspace_id = 'test-workspace'" + ) + .execute(&db) + .await?; + let resp = authed(client().get(format!("{base}/target"))) + .send() + .await?; + assert!(resp.json::().await?["connection"].is_null()); + + // A target that refuses the stored token must not answer 401: the browser reads an + // unhandled 401 as its own session having expired and logs the user out of this instance. + let mc = build_crypt(&db, "test-workspace").await?; + sqlx::query!( + "UPDATE remote_deploy_token SET token = $1 WHERE workspace_id = 'test-workspace'", + encrypt(&mc, "revoked-token") + ) + .execute(&db) + .await?; + let resp = authed(client().get(format!("{base}/proxy/{key}/users/whoami"))) + .send() + .await?; + assert_eq!(resp.status(), 502); + + // Nor does a connect lock anything against a removal from the workspace, so its row can land + // after the removal cleared the table: a row from an earlier membership must not come back + // with a re-add, even one whose `created_at` reads earlier than the connect (it is the start + // of the re-adding transaction). + let as_test2 = + |builder: reqwest::RequestBuilder| builder.header("Authorization", "Bearer SECRET_TOKEN_2"); + let resp = as_test2(client().post(format!("{base}/connect"))) + .json(&json!({"token": "SECRET_TOKEN_2", "target": target})) + .send() + .await?; + assert_eq!(resp.status(), 200); + let key2 = resp.json::().await?["proxy_key"] + .as_str() + .unwrap() + .to_string(); + let resp = as_test2(client().get(format!("{base}/target"))) + .send() + .await?; + assert_eq!( + resp.json::().await?["connection"]["proxy_key"], + key2.as_str() + ); + sqlx::query!( + "UPDATE usr SET created_at = created_at - interval '1 hour' + WHERE workspace_id = 'test-workspace' AND email = 'test2@windmill.dev'" + ) + .execute(&db) + .await?; + let resp = as_test2(client().get(format!("{base}/target"))) + .send() + .await?; + assert!(resp.json::().await?["connection"].is_null()); + let resp = as_test2(client().get(format!("{base}/proxy/{key2}/users/whoami"))) + .send() + .await?; + assert_eq!(resp.status(), 400); + assert!(resp.text().await?.contains("Connect to")); + + // Deleting the account must take its token with it, or the next account created with that + // address would act on the remote as this one. + let resp = authed(client().delete(format!( + "http://localhost:{port}/api/users/delete/test2@windmill.dev" + ))) + .send() + .await?; + assert_eq!(resp.status(), 200); + let left = sqlx::query_scalar!( + "SELECT count(*) FROM remote_deploy_token WHERE email = 'test2@windmill.dev'" + ) + .fetch_one(&db) + .await?; + assert_eq!(left, Some(0)); + + // The token was granted for one target and is never sent to another, so re-pointing the + // workspace leaves the caller unconnected instead of handing its token to the new target. + set_target(format!("http://localhost:{port}"), "other-workspace").await; + let resp = authed(client().get(format!("{base}/target"))) + .send() + .await?; + assert!(resp.json::().await?["connection"].is_null()); + let resp = authed(client().get(format!("{base}/proxy/{key}/users/whoami"))) + .send() + .await?; + assert_eq!(resp.status(), 400); + + // The row for the old target outlived the change, as one from a connect in flight across it + // would: pointing the setting back must not revive it, even with a stamp that reads earlier + // than the connect. + set_target(format!("http://localhost:{port}"), "test-workspace").await; + sqlx::query!( + "UPDATE workspace_settings SET remote_deploy_target_changed_at = '2000-01-01' + WHERE workspace_id = 'test-workspace'" + ) + .execute(&db) + .await?; + let resp = authed(client().get(format!("{base}/target"))) + .send() + .await?; + assert!(resp.json::().await?["connection"].is_null()); + + // A connect in flight across a disconnect must not undo it when it lands. The disconnect is + // stamped an hour ahead, as if every connect starting now had started before it. + let resp = authed(client().post(format!("{base}/connect"))) + .json(&json!({"token": "SECRET_TOKEN", "target": target})) + .send() + .await?; + assert_eq!(resp.status(), 200); + let resp = authed(client().post(format!("{base}/disconnect"))) + .send() + .await?; + assert_eq!(resp.status(), 200); + sqlx::query!( + "UPDATE remote_deploy_token SET connected_at = clock_timestamp() + interval '1 hour' + WHERE workspace_id = 'test-workspace' AND email = 'test@windmill.dev'" + ) + .execute(&db) + .await?; + let resp = authed(client().post(format!("{base}/connect"))) + .json(&json!({"token": "SECRET_TOKEN", "target": target})) + .send() + .await?; + assert_eq!(resp.status(), 400); + let resp = authed(client().get(format!("{base}/target"))) + .send() + .await?; + assert!(resp.json::().await?["connection"].is_null()); + + // A superadmin outside the workspace has no membership to bind to, so the row is bound to the + // account by the credential making the request: an account deleted during the remote call, + // and its address taken by another, must not inherit it. A deleted credential that auth still + // holds in its cache stands in for one whose account went away mid-call. + sqlx::query!( + "DELETE FROM usr WHERE workspace_id = 'test-workspace' AND email = 'test@windmill.dev'" + ) + .execute(&db) + .await?; + sqlx::query!("DELETE FROM remote_deploy_token WHERE email = 'test@windmill.dev'") + .execute(&db) + .await?; + sqlx::query!( + "INSERT INTO token (token_hash, token_prefix, token, email, label, super_admin) + VALUES (encode(sha256('GONE_TOKEN'::bytea), 'hex'), 'GONE_TOKEN', 'GONE_TOKEN', + 'test@windmill.dev', 'gone', true)" + ) + .execute(&db) + .await?; + let as_gone = + |builder: reqwest::RequestBuilder| builder.header("Authorization", "Bearer GONE_TOKEN"); + let resp = as_gone(client().get(format!("{base}/target"))) + .send() + .await?; + assert_eq!(resp.status(), 200); + sqlx::query!("DELETE FROM token WHERE token_prefix = 'GONE_TOKEN'") + .execute(&db) + .await?; + let resp = as_gone(client().post(format!("{base}/connect"))) + .json(&json!({"token": "SECRET_TOKEN", "target": target})) + .send() + .await?; + assert_eq!(resp.status(), 400); + let resp = authed(client().post(format!("{base}/connect"))) + .json(&json!({"token": "SECRET_TOKEN", "target": target})) + .send() + .await?; + assert_eq!(resp.status(), 200); + + Ok(()) +} diff --git a/backend/windmill-api-integration-tests/tests/workspaces.rs b/backend/windmill-api-integration-tests/tests/workspaces.rs index ff603269e7..29b8a0a611 100644 --- a/backend/windmill-api-integration-tests/tests/workspaces.rs +++ b/backend/windmill-api-integration-tests/tests/workspaces.rs @@ -319,6 +319,14 @@ async fn test_workspace_endpoints(db: Pool) -> anyhow::Result<()> { .unwrap(); assert_eq!(resp.status(), 200); + let resp = authed(client().post(format!("{base}/edit_webhook"))) + .json(&json!({"webhook": "http://127.0.0.1/hook"})) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 400); + assert!(resp.text().await?.contains("ALLOW_PRIVATE_WEBHOOK_URLS")); + // --- edit_auto_invite (EE-gated) --- let resp = authed(client().post(format!("{base}/edit_auto_invite"))) .json(&json!({"operator": false, "invite_all": false, "auto_add": false})) diff --git a/backend/windmill-api-jobs/src/execution.rs b/backend/windmill-api-jobs/src/execution.rs index 7184f8ecec..9ad7189434 100644 --- a/backend/windmill-api-jobs/src/execution.rs +++ b/backend/windmill-api-jobs/src/execution.rs @@ -40,8 +40,8 @@ use windmill_common::{ FlowVersionInfo, DB, }; use windmill_queue::{ - cancel_job, get_result_and_success_by_id_from_flow, parse_result_object, push, PushArgs, - PushArgsOwned, PushIsolationLevel, + cancel_job, check_tag_available_for_push, get_result_and_success_by_id_from_flow, + parse_result_object, push, tag_reads_args, PushArgs, PushArgsOwned, PushIsolationLevel, }; use crate::types::RunJobQuery; @@ -50,7 +50,27 @@ use crate::types::RunJobQuery; // Tag / license checks // --------------------------------------------------------------------------- +/// `args` must be the ones the job is pushed with, see [`windmill_queue::check_tag_available_for_push`]. pub async fn check_tag_available_for_workspace( + db: &DB, + w_id: &str, + tag: &Option, + args: &PushArgs<'_>, + authed: &ApiAuthed, +) -> error::Result<()> { + if let Some(tag) = tag.as_deref().filter(|t| !t.is_empty()) { + let tags = get_scope_tags(authed); + // Job-aware: a WM_TOKEN running as a superadmin must not unlock restricted tags. + let is_super_admin = windmill_api_auth::is_super_admin_authed(db, authed).await?; + check_tag_available_for_push(db, w_id, tag, args, is_super_admin, tags).await + } else { + Ok(()) + } +} + +/// [`check_tag_available_for_workspace`] for a tag filled in from values nothing can check yet, +/// such as a flow's preprocessor output: only an entry spelled exactly like it admits it. +pub async fn check_tag_as_written_available_for_workspace( db: &DB, w_id: &str, tag: &Option, @@ -60,7 +80,16 @@ pub async fn check_tag_available_for_workspace( let tags = get_scope_tags(authed); // Job-aware: a WM_TOKEN running as a superadmin must not unlock restricted tags. let is_super_admin = windmill_api_auth::is_super_admin_authed(db, authed).await?; - check_tag_available_for_workspace_internal(db, w_id, tag, is_super_admin, tags).await + check_tag_available_for_workspace_internal( + db, + w_id, + tag, + None, + std::future::ready(w_id.to_string()), + is_super_admin, + tags, + ) + .await } else { Ok(()) } @@ -760,9 +789,25 @@ pub async fn run_flow<'c>( .. } = flow_version_info; - let tag = run_query.tag.clone().or(tag); + let flow_tag = tag; + let tag = run_query.tag.clone().or(flow_tag.clone()); + let push_args = PushArgs { args: &args.args, extra: args.extra }; + let apply_preprocessor = + !run_query.skip_preprocessor.unwrap_or(false) && has_preprocessor.unwrap_or(false); - check_tag_available_for_workspace(&db, &w_id, &tag, &authed).await?; + if !apply_preprocessor || run_query.tag.is_some() { + check_tag_available_for_workspace(&db, &w_id, &tag, &push_args, &authed).await?; + } + // `push` drops the tag of a flow whose preprocessor runs, and the worker then puts the flow on + // its own tag filled in from the preprocessor's output, after this caller's identity is gone. + // So that tag is the one to check, and an `$args[...]` in it is judged as written. + if apply_preprocessor { + if flow_tag.as_deref().is_some_and(tag_reads_args) { + check_tag_as_written_available_for_workspace(&db, &w_id, &flow_tag, &authed).await?; + } else { + check_tag_available_for_workspace(&db, &w_id, &flow_tag, &push_args, &authed).await?; + } + } let scheduled_for = run_query.get_scheduled_for(&db).await?; let return_tx = tx_o.is_some(); @@ -798,11 +843,10 @@ pub async fn run_flow<'c>( path: flow_path.to_string(), dedicated_worker, version, - apply_preprocessor: !run_query.skip_preprocessor.unwrap_or(false) - && has_preprocessor.unwrap_or(false), + apply_preprocessor, labels, }, - PushArgs { args: &args.args, extra: args.extra }, + push_args, authed.display_username(), email, permissioned_as, @@ -988,7 +1032,8 @@ pub async fn push_script_job_by_path_into_queue<'c>( let scheduled_for = run_query.get_scheduled_for(&db).await?; let tag = run_query.tag.clone().or(tag); - check_tag_available_for_workspace(&db, &w_id, &tag, &authed).await?; + let push_args = PushArgs { args: &args.args, extra: args.extra }; + check_tag_available_for_workspace(&db, &w_id, &tag, &push_args, &authed).await?; let return_tx = tx_o.is_some(); @@ -1020,7 +1065,7 @@ pub async fn push_script_job_by_path_into_queue<'c>( tx, &w_id, job_payload, - PushArgs { args: &args.args, extra: args.extra }, + push_args, authed.display_username(), email, permissioned_as, diff --git a/backend/windmill-api-scripts/src/scripts.rs b/backend/windmill-api-scripts/src/scripts.rs index a4667e4876..51f2619805 100644 --- a/backend/windmill-api-scripts/src/scripts.rs +++ b/backend/windmill-api-scripts/src/scripts.rs @@ -55,7 +55,7 @@ use windmill_common::{ min_version_supports_runnable_settings_v0, RunnableSettings, RunnableSettingsTrait, }, scripts::{hash_script, ScriptRunnableSettingsHandle, ScriptRunnableSettingsInline}, - utils::{paginate_without_limits, WarnAfterExt}, + utils::{paginate_optional, paginate_without_limits, WarnAfterExt}, worker::CLOUD_HOSTED, }; use windmill_object_store::upload_artifact_to_store; @@ -2312,6 +2312,20 @@ async fn create_script_internal<'c>( .await?; } + if p_path != &ns.path { + // Everything left at the old path is a draft this deploy didn't + // consume — teammates' rows, and the deployer's own when the caller + // asked us to keep it. Carry them rather than strand them. + windmill_common::user_drafts::move_drafts_for_path( + &mut tx, + &w_id, + &[UserDraftItemKind::Script], + p_path, + &ns.path, + ) + .await?; + } + sqlx::query!( "UPDATE capture_config SET path = $1 WHERE path = $2 AND workspace_id = $3 AND is_flow IS FALSE", ns.path, @@ -2389,19 +2403,32 @@ async fn create_script_internal<'c>( tx = push_scheduled_job(&db, tx, &schedule, None, None).await?; } } - } else if !skip_draft_deletion { - // See the matching branch above — only wipe the deployer's own - // draft (plus the legacy NULL-email row). - sqlx::query!( - "DELETE FROM draft WHERE path = $1 AND workspace_id = $2 AND typ = 'script' \ - AND (email = $3 OR email IS NULL)", - ns.path, - &w_id, - &authed.email, - ) - .execute(&mut *tx) - .await?; + } else { + if !skip_draft_deletion { + // See the matching branch above — only wipe the deployer's own + // draft (plus the legacy NULL-email row). + sqlx::query!( + "DELETE FROM draft WHERE path = $1 AND workspace_id = $2 AND typ = 'script' \ + AND (email = $3 OR email IS NULL)", + ns.path, + &w_id, + &authed.email, + ) + .execute(&mut *tx) + .await?; + } } + // Every deploy, not only a new script: an archived script's draft can be moved away + // (`move_draft` ignores archived rows), and unarchiving redeploys at the same path, + // where a route left behind would send the live script's saves to the moved draft. + windmill_common::user_drafts::clear_draft_moves_from( + &mut tx, + &w_id, + &[UserDraftItemKind::Script], + &ns.path, + p_path_opt.as_deref(), + ) + .await?; if p_hashes.is_some() && !p_hashes.unwrap().is_empty() { audit_log( &mut *tx, @@ -3061,17 +3088,24 @@ async fn get_script_history( authed: ApiAuthed, Extension(user_db): Extension, Path((w_id, path)): Path<(String, StripPath)>, + Query(pagination): Query, ) -> JsonResult> { let path = path.to_path(); check_scopes(&authed, || format!("scripts:read:{}", path))?; + // Unasked-for, this listing stays whole: the deployment-history panels, the restart + // picker and the CLI all read it without paging. The diff picker asks for a page. + let (per_page, offset) = paginate_optional(pagination); let mut tx = user_db.begin(&authed).await?; let query_result = sqlx::query!( - "SELECT s.hash as hash, dm.deployment_msg as deployment_msg, s.created_at as created_at + "SELECT s.hash as hash, dm.deployment_msg as deployment_msg, s.created_at as created_at, s.created_by as created_by FROM script s LEFT JOIN deployment_metadata dm ON s.hash = dm.script_hash WHERE s.workspace_id = $1 AND s.path = $2 - ORDER by s.created_at DESC", + ORDER by s.created_at DESC + LIMIT $3 OFFSET $4", w_id, path, + per_page, + offset, ) .fetch_all(&mut *tx) .await?; @@ -3083,6 +3117,7 @@ async fn get_script_history( script_hash: ScriptHash(row.hash), deployment_msg: row.deployment_msg, created_at: Some(row.created_at), + created_by: Some(row.created_by), }) .collect(); return Ok(Json(result)); @@ -3097,7 +3132,7 @@ async fn get_latest_version( check_scopes(&authed, || format!("scripts:read:{}", path))?; let mut tx = user_db.begin(&authed).await?; let row_o = sqlx::query!( - "SELECT s.hash as hash, dm.deployment_msg as deployment_msg, s.created_at as created_at + "SELECT s.hash as hash, dm.deployment_msg as deployment_msg, s.created_at as created_at, s.created_by as created_by FROM script s LEFT JOIN deployment_metadata dm ON s.hash = dm.script_hash WHERE s.workspace_id = $1 AND s.path = $2 ORDER by s.created_at DESC LIMIT 1", @@ -3113,6 +3148,7 @@ async fn get_latest_version( script_hash: ScriptHash(row.hash), deployment_msg: row.deployment_msg, created_at: Some(row.created_at), + created_by: Some(row.created_by), }; return Ok(Json(Some(result))); } else { diff --git a/backend/windmill-api-settings/src/lib.rs b/backend/windmill-api-settings/src/lib.rs index 3bbaf888e2..e8d8a7124c 100644 --- a/backend/windmill-api-settings/src/lib.rs +++ b/backend/windmill-api-settings/src/lib.rs @@ -945,6 +945,8 @@ async fn run_setting_pre_write_hook( AI_CONFIG_SETTING => { windmill_ai::ai_types::validate_model_pricing_json(value) .map_err(error::Error::BadRequest)?; + windmill_ai::ai_types::validate_token_maps_json(value) + .map_err(error::Error::BadRequest)?; } AUTOMATE_USERNAME_CREATION_SETTING => { if value.as_bool().unwrap_or(false) { @@ -1680,6 +1682,8 @@ struct CustomInstanceDbLogs { replication_user: String, #[serde(skip_serializing_if = "Option::is_none")] replication_user_error: Option, + #[serde(skip_serializing_if = "String::is_empty")] + user_connect: String, } async fn list_custom_instance_pg_databases( @@ -1897,9 +1901,57 @@ async fn setup_custom_instance_pg_database_inner( } } + // Everything above logged in as the DATABASE_URL user, but whatever uses the database logs + // in as custom_instance_user. A proxy that routes on the login name can accept one and + // refuse the other, which would otherwise surface only once a data table first connects. + let user_creds = PgDatabase { + user: Some(windmill_common::datatable_roles::CUSTOM_INSTANCE_USER.to_string()), + password: Some(windmill_common::utils::get_custom_pg_instance_password(db).await?), + ..pg_creds + }; + let (client, connection) = user_creds + .connect(Some(db)) + .await + .map_err(|e| custom_instance_user_connect_error(&user_creds.host, dbname, e))?; + let join_handle = tokio::spawn(async move { connection.await }); + logs.user_connect = "OK".to_string(); + drop(client); // /!\ Drop before joining to avoid deadlock + windmill_common::shutdown_pg_connection(join_handle).await?; + Ok(()) } +fn custom_instance_user_connect_error(host: &str, dbname: &str, e: error::Error) -> error::Error { + let cause = match &e { + error::Error::Anyhow { error, .. } => format!("{error:#}"), + e => e.to_string(), + }; + // Supavisor, Supabase's pooler, reads the tenant to route to from the login + // (`.`), and only knows the logins configured for that tenant. + let lower = cause.to_lowercase(); + let routing_refused = [ + "enoidentifier", + "tenant identifier", + "tenant or user", + "tenant/user", + ] + .iter() + .any(|signature| lower.contains(signature)); + if routing_refused { + error::Error::BadConfig(format!( + "DATABASE_URL reaches Postgres through a connection pooler ({host}) that picks the \ + server to route to from the login name, and it refused custom_instance_user, the \ + role Windmill uses for instance databases ({cause}). Instance databases cannot be \ + used through this pooler: use your own Postgres database instead, or point \ + DATABASE_URL at the Postgres server directly rather than at the pooler." + )) + } else { + error::Error::ExecutionErr(format!( + "Could not connect to {dbname} as custom_instance_user: {cause}" + )) + } +} + async fn drop_custom_instance_pg_database( authed: ApiAuthed, Extension(db): Extension, @@ -2302,6 +2354,34 @@ mod tests { use std::collections::BTreeMap; use windmill_common::instance_config::{GlobalSettings, InstanceConfig, WorkerGroupConfig}; + #[test] + fn supavisor_refusing_custom_instance_user_is_named() { + use windmill_common::error::{to_anyhow, Error}; + let connect_error = |message: &str| { + let e = Error::from(to_anyhow(std::io::Error::other(message.to_string()))); + super::custom_instance_user_connect_error( + "aws-0-eu-west-1.pooler.supabase.com", + "dt", + e, + ) + }; + for supavisor in [ + "db error: FATAL: (ENOIDENTIFIER) no tenant identifier provided", + "db error: FATAL: Tenant or user not found", + ] { + assert!( + matches!(connect_error(supavisor), Error::BadConfig(m) if m.contains("connection pooler")), + "{supavisor}" + ); + } + assert!(matches!( + connect_error( + "db error: FATAL: password authentication failed for user \"custom_instance_user\"" + ), + Error::ExecutionErr(_) + )); + } + #[test] fn instance_config_yaml_round_trip() { let config = InstanceConfig { diff --git a/backend/windmill-api-users/src/users.rs b/backend/windmill-api-users/src/users.rs index da81c7e228..a1b1054a25 100644 --- a/backend/windmill-api-users/src/users.rs +++ b/backend/windmill-api-users/src/users.rs @@ -171,10 +171,7 @@ pub fn make_unauthed_service() -> Router { .route("/logout", post(logout).get(logout)) .route("/is_first_time_setup", get(is_first_time_setup)) .route("/request_password_reset", post(request_password_reset)) - .route( - "/login_link/{token}", - get(consume_login_link).post(confirm_login_link), - ) + .route("/login_link/{token}", get(consume_login_link)) .route("/is_smtp_configured", get(is_smtp_configured)) .route( "/is_password_login_disabled", @@ -530,7 +527,7 @@ async fn list_users_as_super_admin( let rows = if active_only.is_some_and(|x| x) { sqlx::query_as!( GlobalUserInfo, - r#"WITH active_users AS (SELECT distinct username as email FROM (SELECT username, timestamp, operation FROM audit_partitioned UNION ALL SELECT username, timestamp, operation FROM audit) AS a WHERE timestamp > NOW() - INTERVAL '1 month' AND (operation = 'users.login' OR operation = 'oauth.login' OR operation = 'users.token.refresh')), + r#"WITH active_users AS (SELECT distinct username as email FROM audit_partitioned WHERE timestamp > NOW() - INTERVAL '1 month' AND (operation = 'users.login' OR operation = 'oauth.login' OR operation = 'users.token.refresh')), authors as (SELECT distinct email FROM usr WHERE usr.operator IS false) SELECT email as "email!", (email NOT IN (SELECT email FROM authors)) as operator_only, NULL::bool as is_workspace_admin, login_type::text, verified as "verified!", super_admin as "super_admin!", devops as "devops!", name, company, username, first_time_user as "first_time_user!", role_source as "role_source!", disabled as "disabled!", NULL::text as workspace_id FROM password @@ -2566,6 +2563,14 @@ pub async fn delete_workspace_user_internal( .execute(&mut **tx) .await?; + sqlx::query!( + "DELETE FROM remote_deploy_token WHERE email = $1 AND workspace_id = $2", + email_to_delete, + w_id + ) + .execute(&mut **tx) + .await?; + sqlx::query!( "DELETE FROM usr_to_group WHERE usr = $1 AND workspace_id = $2", username_to_delete, @@ -3287,12 +3292,9 @@ async fn impersonate( } const LOGIN_LINK_DEFAULT_TTL_S: u32 = 600; -// Long enough for a link sent by email to still work when it is read. `require_login_type` is -// only checked at mint, so a much longer cap would need re-checking it when the link is opened. -const LOGIN_LINK_MAX_TTL_S: u32 = 7200; +const LOGIN_LINK_MAX_TTL_S: u32 = 900; const LOGIN_LINK_DEFAULT_RD: &str = "/user/workspaces"; const LOGIN_LINK_EXPIRED_PAGE: &str = "/user/login_link_expired"; -const LOGIN_LINK_CONFIRM_PAGE: &str = "/user/login_link"; #[derive(Deserialize)] pub struct NewLoginLink { @@ -3303,9 +3305,6 @@ pub struct NewLoginLink { /// account it created can require `pending_oauth`, so the link stops working once the /// owner has set a password or signed in with a provider. pub require_login_type: Option, - /// Hand out a page that signs in only when its button is clicked. Mail scanners open links - /// on delivery, and opening the plain link spends it, so a link sent by email sets this. - pub confirm: Option, } #[derive(Serialize)] @@ -3456,12 +3455,11 @@ async fn create_login_link( .await?; tx.commit().await?; - let base_url = (**BASE_URL.load()).clone(); - let url = if nl.confirm.unwrap_or(false) { - format!("{base_url}{LOGIN_LINK_CONFIRM_PAGE}?token={token}") - } else { - format!("{base_url}/api/auth/login_link/{token}") - }; + let url = format!( + "{}/api/auth/login_link/{}", + (**BASE_URL.load()).clone(), + token + ); Ok((StatusCode::CREATED, Json(LoginLink { url, expires_at }))) } @@ -3707,45 +3705,19 @@ async fn consume_login_link( Path(token): Path, Query(query): Query, ) -> Result { - let location = redeem_login_link(&headers, cookies, &db, &token, query.rd).await?; - Ok(login_link_redirect(location)) -} - -#[derive(Serialize)] -struct LoginLinkLocation { - location: String, -} - -/// The confirmation page's click. It answers with where to go rather than redirecting, and the -/// page navigates there itself. -async fn confirm_login_link( - headers: axum::http::HeaderMap, - cookies: Cookies, - Extension(db): Extension, - Path(token): Path, -) -> JsonResult { - let location = redeem_login_link(&headers, cookies, &db, &token, None).await?; - Ok(Json(LoginLinkLocation { location })) -} - -/// Spends the link and sets the session cookie, returning the post-login destination; or -/// returns the explanation page, with no session, when the link cannot be used. -async fn redeem_login_link( - headers: &axum::http::HeaderMap, - cookies: Cookies, - db: &DB, - token: &str, - requested_rd: Option, -) -> Result { - let bounce = |reason: &str| Ok(format!("{LOGIN_LINK_EXPIRED_PAGE}?reason={reason}")); + let bounce = |reason: &str| { + Ok(login_link_redirect(format!( + "{LOGIN_LINK_EXPIRED_PAGE}?reason={reason}" + ))) + }; if token.len() != 32 { return bounce("invalid"); } - let t_hash = hash_token(token); + let t_hash = hash_token(&token); // The account is unknown until the row is read, so only the global and per-IP tiers // apply here; a 32-char random token leaves nothing for the per-account tier to guard. windmill_common::login_rate_limit::check_and_increment_login_attempt( - headers, + &headers, &t_hash[..TOKEN_PREFIX_LEN], )?; @@ -3812,10 +3784,11 @@ async fn redeem_login_link( .await?; tx.commit().await?; - Ok(link + let rd = link .rd - .or_else(|| same_origin_rd(requested_rd)) - .unwrap_or_else(|| LOGIN_LINK_DEFAULT_RD.to_string())) + .or_else(|| same_origin_rd(query.rd)) + .unwrap_or_else(|| LOGIN_LINK_DEFAULT_RD.to_string()); + Ok(login_link_redirect(rd)) } #[derive(Deserialize)] @@ -4050,6 +4023,7 @@ async fn update_token_label( AND NOT starts_with(label, 'sdk_app:') AND NOT starts_with(label, 'impersonation:') AND NOT starts_with(label, 'cli-login:') + AND NOT starts_with(label, 'remote-deploy:') )) RETURNING token_prefix", req.label.as_deref(), @@ -4096,13 +4070,25 @@ async fn leave_workspace( &format!("u/{}", authed.username), ) .await?; - sqlx::query!( + let left = sqlx::query!( "DELETE FROM usr WHERE workspace_id = $1 AND username = $2", &w_id, authed.username ) .execute(&mut *tx) - .await?; + .await? + .rows_affected(); + // A superadmin deploys from workspaces it is no member of; leaving none is no reason to drop + // its connection. + if left > 0 { + sqlx::query!( + "DELETE FROM remote_deploy_token WHERE email = $1 AND workspace_id = $2", + &authed.email, + &w_id + ) + .execute(&mut *tx) + .await?; + } audit_log( &mut *tx, diff --git a/backend/windmill-api-workers/src/lib.rs b/backend/windmill-api-workers/src/lib.rs index 07bef364c8..0102f1b033 100644 --- a/backend/windmill-api-workers/src/lib.rs +++ b/backend/windmill-api-workers/src/lib.rs @@ -436,11 +436,6 @@ async fn get_workspace_fairness_events( WITH capped AS ( SELECT timestamp, operation, resource, parameters FROM audit_partitioned - WHERE workspace_id = 'admins' - AND operation = 'workspace_fairness.capped' - UNION ALL - SELECT timestamp, operation, resource, parameters - FROM audit WHERE workspace_id = 'admins' AND operation = 'workspace_fairness.capped' ORDER BY timestamp DESC @@ -448,11 +443,6 @@ async fn get_workspace_fairness_events( ), uncapped AS ( SELECT timestamp, operation, resource, parameters FROM audit_partitioned - WHERE workspace_id = 'admins' - AND operation = 'workspace_fairness.uncapped' - UNION ALL - SELECT timestamp, operation, resource, parameters - FROM audit WHERE workspace_id = 'admins' AND operation = 'workspace_fairness.uncapped' ORDER BY timestamp DESC diff --git a/backend/windmill-api-workspaces/Cargo.toml b/backend/windmill-api-workspaces/Cargo.toml index 52a58aa1a3..7ca1f692ad 100644 --- a/backend/windmill-api-workspaces/Cargo.toml +++ b/backend/windmill-api-workspaces/Cargo.toml @@ -35,6 +35,7 @@ windmill-store.workspace = true axum.workspace = true chrono.workspace = true +constant_time_eq.workspace = true futures = { workspace = true, optional = true } hex.workspace = true magic-crypt.workspace = true @@ -42,6 +43,7 @@ http.workspace = true hyper.workspace = true lazy_static.workspace = true regex.workspace = true +reqwest.workspace = true serde.workspace = true serde_json.workspace = true sha2.workspace = true @@ -49,5 +51,6 @@ sqlx.workspace = true tokio.workspace = true tokio-postgres.workspace = true tracing.workspace = true +url.workspace = true uuid.workspace = true strum.workspace = true diff --git a/backend/windmill-api-workspaces/src/datatable_acl.rs b/backend/windmill-api-workspaces/src/datatable_acl.rs new file mode 100644 index 0000000000..04dc3d4e80 --- /dev/null +++ b/backend/windmill-api-workspaces/src/datatable_acl.rs @@ -0,0 +1,2034 @@ +/* + * Author: Ruben Fiszel + * Copyright: Windmill Labs, Inc 2022 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + +//! Ownership and grants on the objects of an instance data table. +//! +//! [`datatable_permissions`](crate::datatable_permissions) decides who may connect as which role; +//! this decides what each role may then touch. Every change is a real `GRANT`, `REVOKE`, +//! `ALTER ... OWNER TO` or `ALTER DEFAULT PRIVILEGES`, so Postgres is what enforces it. +//! +//! Reading is open to anyone who reaches the data table. Planning and applying are for those who +//! administer it — admins of the workspace that governs it, and superadmins. All of it is +//! Enterprise Edition ([`crate::datatable_acl_oss`]). + +use std::collections::BTreeMap; + +use axum::{ + extract::{Extension, Path, Query}, + routing::{get, post}, + Json, Router, +}; +use serde::{Deserialize, Serialize}; +use tokio::sync::mpsc; +use tokio_postgres::error::{DbError, SqlState}; +use tokio_postgres::AsyncMessage; + +use windmill_api_auth::ApiAuthed; +use windmill_audit::audit_oss::audit_log; +use windmill_audit::ActionKind; +use windmill_common::datatable_roles::{ + lock_role_catalog, quote_ident, read_role_catalog, read_role_catalog_tx, DatatableRoleCatalog, + ADMIN_DATATABLE_ROLE, CUSTOM_INSTANCE_USER, +}; +use windmill_common::error::{pg_error_message, Error, JsonResult, Result}; +use windmill_common::workspaces::{resolve_governing_datatable, DataTable, GoverningDatatable}; +use windmill_common::{PgDatabase, DB}; + +use crate::datatable_permissions::{ensure_governs_datatable, ensure_reaches_governing_datatable}; + +pub(crate) fn routes() -> Router { + Router::new() + .route("/datatable_acl/{datatable_name}", get(get_datatable_acl)) + .route( + "/datatable_acl/{datatable_name}/plan", + post(plan_datatable_acl), + ) + .route( + "/datatable_acl/{datatable_name}/apply", + post(apply_datatable_acl), + ) +} + +/// What a read or a change is about. +#[derive(Deserialize, Serialize, Debug, Clone, PartialEq)] +#[serde(tag = "kind", rename_all = "snake_case")] +pub enum AclTarget { + /// The data table's own database — where the privilege to create schemas lives. + Database, + Schema { + schema: String, + }, + Table { + schema: String, + table: String, + }, +} + +impl AclTarget { + /// The schema the target is in, absent for the database itself. + pub(crate) fn schema(&self) -> Option<&str> { + match self { + AclTarget::Database => None, + AclTarget::Schema { schema } => Some(schema), + AclTarget::Table { schema, .. } => Some(schema), + } + } + + /// What it is called in a message. + pub(crate) fn label(&self, dbname: &str) -> String { + match self { + AclTarget::Database => dbname.to_string(), + AclTarget::Schema { schema } => schema.clone(), + AclTarget::Table { schema, table } => format!("{schema}.{table}"), + } + } +} + +#[derive(Deserialize, Debug)] +pub struct AclTargetQuery { + kind: String, + schema: Option, + table: Option, +} + +impl TryFrom for AclTarget { + type Error = Error; + fn try_from(q: AclTargetQuery) -> Result { + match (q.kind.as_str(), q.schema, q.table) { + ("database", _, _) => Ok(AclTarget::Database), + ("schema", Some(schema), _) => Ok(AclTarget::Schema { schema }), + ("table", Some(schema), Some(table)) => Ok(AclTarget::Table { schema, table }), + ("schema" | "table", None, _) => { + Err(Error::BadRequest("This target needs a schema".to_string())) + } + ("table", _, None) => Err(Error::BadRequest( + "A table target needs a table".to_string(), + )), + (kind, _, _) => Err(Error::BadRequest(format!("Unknown ACL target '{kind}'"))), + } + } +} + +/// Where a set of privileges applies, relative to the target. +/// +/// `Future*` covers what does not exist yet: those become `ALTER DEFAULT PRIVILEGES`, which only +/// binds objects created by the roles it names. +#[derive(Deserialize, Serialize, Debug, Clone, Copy, PartialEq)] +#[serde(rename_all = "snake_case")] +pub enum GrantScope { + /// The target itself — the database, the schema, or the table. + Target, + AllTables, + AllSequences, + AllFunctions, + FutureTables, + FutureSequences, + FutureFunctions, +} + +impl GrantScope { + pub(crate) fn is_future(&self) -> bool { + matches!( + self, + GrantScope::FutureTables | GrantScope::FutureSequences | GrantScope::FutureFunctions + ) + } +} + +/// A change to plan. One at a time: each is confirmed against its own SQL. +#[derive(Deserialize, Serialize, Debug, Clone)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum AclChange { + /// Hand the target — and, for a schema, everything already in it but an extension's members, + /// which stay with the extension — to another role. + SetOwner { + role: String, + }, + Grant { + role: String, + privileges: Vec, + scope: GrantScope, + }, + Revoke { + role: String, + privileges: Vec, + scope: GrantScope, + /// Objects inside the target, empty for the target itself. `ON ALL TABLES` grants read + /// back per object, so they are revoked per object — and the same privileges on several + /// of them are revoked together. + #[serde(default)] + objects: Vec, + }, +} + +impl AclChange { + /// The role the change is about, as the editor names it. + fn role(&self) -> &str { + match self { + AclChange::SetOwner { role } + | AclChange::Grant { role, .. } + | AclChange::Revoke { role, .. } => role, + } + } +} + +#[derive(Deserialize, Debug)] +pub struct AclChangeRequest { + pub target: AclTarget, + pub change: AclChange, + /// The statements the plan showed. An apply runs only those: it plans again and refuses if the + /// result differs. + #[serde(default)] + pub statements: Option>, +} + +/// An object inside a schema, named the way `REVOKE ... ON ` needs it. +#[derive(Deserialize, Serialize, Debug, Clone, PartialEq)] +pub struct AclObject { + pub name: String, + /// `TABLE`, `SEQUENCE`, `FUNCTION`, `PROCEDURE` or `TYPE`: what the object is. + /// [`object_keyword`] turns it into the keyword a revoke takes; a type has none. + pub kind: String, + /// A routine is identified by its argument types, not by its name: two `f` in one schema are + /// two objects. Absent for everything else. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub args: Option, +} + +/// A grant as the database has it, under the role names the editor uses. +#[derive(Serialize, Debug, PartialEq)] +pub struct AclGrant { + /// A data table role's name, `admin` for `custom_instance_user`, else the raw Postgres role + /// (`PUBLIC` included). + pub grantee: String, + pub privileges: Vec, + /// `None` for the target itself, else the object inside it. + #[serde(skip_serializing_if = "Option::is_none")] + pub object: Option, + /// `TABLES` / `SEQUENCES` / `FUNCTIONS` / `TYPES` (or, on the database, `SCHEMAS`) when this + /// is a default privilege, which applies to objects that do not exist yet. + #[serde(skip_serializing_if = "Option::is_none")] + pub future: Option, + /// Where the grant comes from, each role once: who granted it, or for a default privilege the + /// role whose future objects it covers. A revoke of some of the privileges takes them back from + /// every source that gave them. + pub sources: Vec, +} + +/// One role a grant comes from. +#[derive(Serialize, Debug, PartialEq)] +pub struct AclSource { + /// Under the same names as [`AclGrant::grantee`]. + pub role: String, + /// What `role` gave of the grant's privileges. A revoke is held back only by a source out of + /// reach that gave some of what it takes back. + pub privileges: Vec, + /// Whether this data table's connection can take back what `role` gave: on an object only the + /// owner's grants when it acts for the owner, or else its own (`grant_source!`); for a default + /// privilege, a creating role it acts for. What a source out of reach gave is not revocable + /// from here; privileges only other sources gave still are. + pub reachable: bool, +} + +#[derive(Serialize, Debug)] +pub struct DatatableAclInfo { + /// Under the same names as [`AclGrant::grantee`]. + pub owner: String, + /// The roles a change may name: `admin`, then every role of the instance catalog. Only for a + /// caller who may change anything, as the catalog is in the permissions drawer. + pub roles: Vec, + /// Whether this caller may plan and apply changes: they administer the data table, on an + /// edition that has the planner. + pub editable: bool, + /// Whether the data table is a clone, whose grants stay as they were copied from its source. + pub clone: bool, + /// Whether the server is Postgres 17 or later, which added the `MAINTAIN` table privilege. + pub supports_maintain: bool, + /// The database the target lives in, which no target carries itself. + pub dbname: String, + pub grants: Vec, + /// What the target holds that is a target of its own: a database's schemas, a schema's tables. + pub children: Vec, +} + +#[derive(Serialize, Debug)] +pub struct AclPlan { + pub statements: Vec, + pub warnings: Vec, +} + +/// The Postgres role a role name stands for. A data table role is a login named exactly like the +/// role, so this is the identity — except `admin`, which is `custom_instance_user`. +/// +/// Anything else is refused, never resolved to some default: every statement a plan writes names +/// the role it is about. +pub(crate) fn pg_role_of(name: &str, catalog: &DatatableRoleCatalog) -> Result { + if name == ADMIN_DATATABLE_ROLE { + return Ok(CUSTOM_INSTANCE_USER.to_string()); + } + if catalog.values().any(|r| r.name == name) { + return Ok(name.to_string()); + } + Err(Error::BadRequest(format!( + "'{name}' is not a data table role of this instance" + ))) +} + +/// The reverse of [`pg_role_of`], for display. A role that is not a data table role reads back as +/// itself. +pub(crate) fn role_name_of(pg_role: &str) -> String { + if pg_role == CUSTOM_INSTANCE_USER { + ADMIN_DATATABLE_ROLE.to_string() + } else { + pg_role.to_string() + } +} + +/// Every role a change may name: `admin` first, then the catalog. +fn role_names(catalog: &DatatableRoleCatalog) -> Vec { + let mut names: Vec = catalog.values().map(|r| r.name.clone()).collect(); + names.sort(); + names.insert(0, ADMIN_DATATABLE_ROLE.to_string()); + names +} + +fn ensure_instance(governing: &GoverningDatatable) -> Result<()> { + if governing.is_instance() { + return Ok(()); + } + Err(Error::BadRequest(format!( + "Data table '{}' is backed by a Postgres resource, so its access is managed on that \ + server directly. Only a data table on the Windmill instance's own database has data \ + table roles to grant to.", + governing.name + ))) +} + +/// The data table's `admin` connection, and the notices Postgres sends on it. +/// +/// Authorization: connects as `custom_instance_user` with the instance's own credentials and checks +/// nothing. Callers MUST have authorized the request first — a request about to be refused must +/// not get as far as this connection. +async fn connect_as_admin_unchecked( + db: &DB, + governing: &GoverningDatatable, +) -> Result<( + tokio_postgres::Client, + mpsc::UnboundedReceiver, + String, +)> { + ensure_instance(governing)?; + // Built from the authorized entry, never by resolving the settings again: a save in between + // could point the entry at a resource on another server and back, and this connection would + // then alter a database the later checks of the entry never see. + let mut pg = PgDatabase::parse_uri(&windmill_common::get_database_url().await?.as_str().await)?; + pg.dbname = governing + .datatable + .database + .as_ref() + .expect("a governing entry owns a database") + .resource_path + .clone(); + pg.user = Some(CUSTOM_INSTANCE_USER.to_string()); + pg.password = Some(windmill_common::utils::get_custom_pg_instance_password(db).await?); + let dbname = pg.dbname.clone(); + let (client, notices) = connect_with_notices(db, &pg).await?; + Ok((client, notices, dbname)) +} + +/// A connection to `pg`, and the notices Postgres sends on it — which is where a grant or revoke +/// that changed nothing is reported ([`execute_acl_statements`]). +pub(crate) async fn connect_with_notices( + db: &DB, + pg: &PgDatabase, +) -> Result<(tokio_postgres::Client, mpsc::UnboundedReceiver)> { + let (client, connection) = pg.connect(Some(db)).await?; + Ok(( + client, + drive_with_notices(connection, windmill_common::TokioPgConnection::poll_message), + )) +} + +type PollMessage = + fn( + &mut C, + &mut std::task::Context<'_>, + ) -> std::task::Poll>>; + +/// Drive `connection` in the background with `poll`, forwarding its notices. +pub(crate) fn drive_with_notices( + mut connection: C, + poll: PollMessage, +) -> mpsc::UnboundedReceiver { + // Unbounded: the driver must never wait on the receiver, which only drains once the statement + // the driver is carrying has completed. + let (notices_tx, notices) = mpsc::unbounded_channel(); + tokio::spawn(async move { + loop { + match std::future::poll_fn(|cx| poll(&mut connection, cx)).await { + Some(Ok(AsyncMessage::Notice(notice))) => { + let _ = notices_tx.send(notice); + } + Some(Ok(_)) => {} + Some(Err(e)) => { + tracing::error!("Datatable ACL connection error: {e}"); + break; + } + None => break, + } + } + }); + notices +} + +/// Run `statements` in order on `tx`, failing on the first that errors or that Postgres only warns +/// about. A privilege the connection cannot pass on is a warning to Postgres (`01007` / `01006`), +/// which then carries on having changed nothing; returning drops the transaction, rolling back +/// everything before it. +/// +/// Authorization: none. Runs `statements` as the connection `tx` is on; callers MUST have authorized +/// changing that database's access, and built the statements themselves. +pub(crate) async fn execute_acl_statements( + tx: &tokio_postgres::Transaction<'_>, + notices: &mut mpsc::UnboundedReceiver, + statements: &[String], +) -> Result<()> { + while notices.try_recv().is_ok() {} + for statement in statements { + tx.batch_execute(statement).await.map_err(|e| { + Error::ExecutionErr(format!( + "Failed to run `{statement}`: {}", + pg_error_message(&e) + )) + })?; + while let Ok(notice) = notices.try_recv() { + if *notice.code() == SqlState::WARNING_PRIVILEGE_NOT_GRANTED + || *notice.code() == SqlState::WARNING_PRIVILEGE_NOT_REVOKED + { + return Err(Error::ExecutionErr(format!( + "`{statement}` did not take effect ({}), so nothing was applied", + notice.message() + ))); + } + } + } + Ok(()) +} + +/// An object whose ownership follows the schema's. +#[derive(Debug, PartialEq)] +pub(crate) struct OwnedObject { + /// The keyword `ALTER ... OWNER TO` takes for this kind of object. + pub(crate) keyword: &'static str, + /// How Postgres names the object (`pg_identify_object`): schema-qualified, quoted where + /// needed, with a routine's arguments or an operator class's access method. It goes into the + /// statement as it is. + pub(crate) identity: String, +} + +/// Everything a schema's change of owner takes along, in schema `$1`, as (kind, identity, owner +/// oid), the first two as `pg_identify_object` gives them. The plan, the check of what this +/// connection may move and the check after the move all read this one list, so what is moved and +/// what is checked cannot differ. +/// +/// Read from what depends on the schema rather than catalog by catalog, so no kind of object is +/// left out by omission; array types, row types and indexes depend on another object instead. +/// Left out: an object's internal parts (a range type's constructors and multirange, an identity +/// column's sequence), a sequence tied to a column (it follows its table, and refuses an owner of +/// its own), an extension's members, and what has no `ALTER ... OWNER` at all (an extension, a text +/// search parser or template). Postgres records no owner for the bootstrap superuser, oid 10. +macro_rules! schema_owned_objects { + () => { + "SELECT o.type AS kind, o.identity, COALESCE(s.refobjid, 10::oid) AS owner + FROM pg_depend d + CROSS JOIN LATERAL pg_identify_object(d.classid, d.objid, 0) o + LEFT JOIN pg_shdepend s + ON s.dbid = (SELECT oid FROM pg_database WHERE datname = current_database()) + AND s.classid = d.classid AND s.objid = d.objid AND s.deptype = 'o' + WHERE d.refclassid = 'pg_namespace'::regclass AND d.deptype = 'n' + AND d.refobjid = (SELECT oid FROM pg_namespace WHERE nspname = $1) + AND d.classid <> ALL(ARRAY['pg_extension'::regclass, 'pg_ts_parser'::regclass, + 'pg_ts_template'::regclass]::oid[]) + AND NOT EXISTS ( + SELECT 1 FROM pg_depend x + WHERE x.classid = d.classid AND x.objid = d.objid AND x.objsubid = 0 + AND (x.deptype IN ('i', 'e') + OR (x.deptype = 'a' AND d.classid = 'pg_class'::regclass + AND x.refobjsubid <> 0)))" + }; +} +#[allow(unused_imports)] +pub(crate) use schema_owned_objects; + +/// The keyword `ALTER ... OWNER TO` takes for a kind of object, as `pg_identify_object` names the +/// kind. A kind missing here is refused rather than skipped, which would leave it behind. +pub(crate) fn owned_keyword(kind: &str) -> Option<&'static str> { + Some(match kind { + "table" => "TABLE", + "view" => "VIEW", + "materialized view" => "MATERIALIZED VIEW", + "sequence" => "SEQUENCE", + "foreign table" => "FOREIGN TABLE", + "type" => "TYPE", + "function" | "procedure" | "aggregate" => "ROUTINE", + "collation" => "COLLATION", + "conversion" => "CONVERSION", + "operator" => "OPERATOR", + "operator class" => "OPERATOR CLASS", + "operator family" => "OPERATOR FAMILY", + "statistics object" => "STATISTICS", + "text search dictionary" => "TEXT SEARCH DICTIONARY", + "text search configuration" => "TEXT SEARCH CONFIGURATION", + _ => return None, + }) +} + +async fn read_owned_objects( + client: &tokio_postgres::Client, + schema: &str, +) -> Result> { + let rows = client + .query( + concat!( + "SELECT kind, identity FROM (", + schema_owned_objects!(), + ") o ORDER BY kind, identity" + ), + &[&schema], + ) + .await + .map_err(|e| { + Error::internal_err(format!( + "Failed to list the objects of schema '{schema}': {}", + pg_error_message(&e) + )) + })?; + rows.into_iter() + .map(|row| { + let kind: &str = row.get(0); + let identity: String = row.get(1); + match owned_keyword(kind) { + Some(keyword) => Ok(OwnedObject { keyword, identity }), + None => Err(Error::BadRequest(format!( + "{identity} is a {kind}, whose owner cannot be changed from here, and schema \ + {schema} would change hands without it. Move it to another schema first." + ))), + } + }) + .collect() +} + +/// Default privileges set database-wide (no `IN SCHEMA`), as `(defaclrole, defaclobjtype, grantee, +/// privilege_type)`. Postgres stores such an entry as a whole acl, the creator's own privileges +/// included, so only what goes beyond the built-in default is a grant. It applies in every schema +/// on top of the schema's own defaults, which cannot take it back. +macro_rules! database_wide_defaults { + () => { + "SELECT d.defaclrole, d.defaclobjtype, a.grantee, a.privilege_type + FROM pg_default_acl d, aclexplode(d.defaclacl) a + WHERE d.defaclnamespace = 0 + AND NOT EXISTS ( + SELECT 1 FROM aclexplode(acldefault( + CASE d.defaclobjtype WHEN 'S' THEN 's' ELSE d.defaclobjtype END, d.defaclrole)) x + WHERE x.grantee = a.grantee AND x.privilege_type = a.privilege_type)" + }; +} + +/// `TABLES`, `SEQUENCES`, `FUNCTIONS` or `TYPES`, as a `defaclobjtype` names them. +fn default_objects_keyword(objtype: &str) -> &'static str { + match objtype { + "r" => "TABLES", + "S" => "SEQUENCES", + "T" => "TYPES", + _ => "FUNCTIONS", + } +} + +/// What a schema's owner holds on what gets created there later. A change of owner hands the new +/// owner the same and takes these back: otherwise every former owner keeps reaching whatever the +/// other roles create there. +#[derive(Debug, PartialEq)] +pub(crate) struct FormerOwnerDefaults { + pub(crate) pg_role: String, + /// (creating role, `TABLES` / `SEQUENCES` / `FUNCTIONS` / `TYPES`), one per default privilege + /// it holds. + pub(crate) defaults: Vec<(String, &'static str)>, +} + +/// The default privileges schema `schema`'s owner holds there — `None` when it holds none, or is +/// `new_owner` already. Refused when one was set by a role this connection cannot act for: only +/// a member of the creating role may change its defaults, so the revoke would fail at apply. Also +/// refused when the owner holds defaults set database-wide, which no change to this schema takes +/// back. +async fn read_former_owner_defaults( + client: &tokio_postgres::Client, + schema: &str, + new_owner: &str, +) -> Result> { + let database_wide = client + .query_opt( + concat!( + "SELECT pg_get_userbyid(n.nspowner), pg_get_userbyid(g.defaclrole), + g.defaclobjtype::text + FROM pg_namespace n, (", + database_wide_defaults!(), + ") g + WHERE n.nspname = $1 AND g.grantee = n.nspowner + AND g.defaclobjtype IN ('r', 'S', 'f', 'T') + AND n.nspowner <> (SELECT oid FROM pg_roles WHERE rolname = $2) + ORDER BY 2, 3 + LIMIT 1" + ), + &[&schema, &new_owner], + ) + .await + .map_err(|e| { + Error::internal_err(format!( + "Failed to read the database-wide default privileges: {}", + pg_error_message(&e) + )) + })?; + if let Some(row) = database_wide { + let pg_role: String = row.get(0); + let creator: String = row.get(1); + return Err(Error::BadRequest(format!( + "{} holds default privileges set database-wide by {}, which no default of schema \ + {schema} takes back, so it would keep reaching what is created here after the change \ + of owner. Revoke them database-wide first: ALTER DEFAULT PRIVILEGES FOR ROLE {} \ + REVOKE ALL PRIVILEGES ON {} FROM {}", + role_name_of(&pg_role), + role_name_of(&creator), + quote_ident(&creator), + default_objects_keyword(row.get(2)), + quote_ident(&pg_role) + ))); + } + + let rows = client + .query( + "SELECT DISTINCT pg_get_userbyid(n.nspowner), pg_get_userbyid(d.defaclrole), + d.defaclobjtype::text, pg_has_role(d.defaclrole, 'USAGE') + FROM pg_namespace n + JOIN pg_default_acl d ON d.defaclnamespace = n.oid + CROSS JOIN LATERAL aclexplode(d.defaclacl) a + WHERE n.nspname = $1 AND a.grantee = n.nspowner + -- What the owner gives itself is about the objects it creates, not about owning + -- the schema, so a move leaves it alone. + AND a.grantee <> d.defaclrole + AND d.defaclobjtype IN ('r', 'S', 'f', 'T') + AND n.nspowner <> (SELECT oid FROM pg_roles WHERE rolname = $2) + ORDER BY 2, 3", + &[&schema, &new_owner], + ) + .await + .map_err(|e| { + Error::internal_err(format!( + "Failed to read the default privileges of schema '{schema}': {}", + pg_error_message(&e) + )) + })?; + let Some(first) = rows.first() else { + return Ok(None); + }; + let pg_role: String = first.get(0); + let plural = |row: &tokio_postgres::Row| default_objects_keyword(row.get(2)); + if let Some(row) = rows.iter().find(|row| !row.get::<_, bool>(3)) { + let creator: String = row.get(1); + return Err(Error::BadRequest(format!( + "{} holds default privileges in schema {schema} from {creator}, which this data \ + table's connection cannot act for, so they would outlive the change of owner. \ + Revoke them as {creator} first: ALTER DEFAULT PRIVILEGES FOR ROLE {} IN SCHEMA {} \ + REVOKE ALL PRIVILEGES ON {} FROM {}", + role_name_of(&pg_role), + quote_ident(&creator), + quote_ident(schema), + plural(row), + quote_ident(&pg_role) + ))); + } + Ok(Some(FormerOwnerDefaults { + defaults: rows + .iter() + .map(|row| (row.get::<_, String>(1), plural(row))) + .collect(), + pg_role, + })) +} + +/// What the catalog holds that a plan depends on, read before planning so the planner stays pure. +#[derive(Debug, Default, PartialEq)] +pub(crate) struct CatalogFacts { + /// Every role that may create objects here, but the one the change is about. + pub(crate) other_pg_roles: Vec, + /// For a schema's change of owner: what moves along with it. + pub(crate) existing_objects: Vec, + /// For a schema's change of owner: the defaults it takes back from the owner it replaces. + pub(crate) former_owner: Option, + /// For a revoke: each grant it takes back. + pub(crate) revoked_grants: Vec, +} + +/// A grant a revoke takes back, as the catalog records it: what `source` gave on `object` (the +/// target itself when `None`), or for a default privilege on what `source` creates later. +#[derive(Debug, PartialEq)] +pub(crate) struct RevokedGrant { + pub(crate) object: Option, + /// The role that made the grant: its grantor, or the creating role of a default privilege. + pub(crate) source: String, + pub(crate) privileges: Vec, +} + +/// An `aclexplode` row's source and whether this connection can take back what it gave, on an +/// object owned by `$owner`. A REVOKE speaks for the grantor Postgres picks itself — the owner, +/// when the connection acts for the owner, and otherwise the connection — and `GRANTED BY` names +/// nobody else, so a grant any other role made stays whatever the connection runs. +macro_rules! grant_source { + ($owner:literal) => { + concat!( + "pg_get_userbyid(a.grantor), a.grantor = CASE WHEN pg_has_role(", + $owner, + ", 'USAGE') THEN ", + $owner, + " ELSE (SELECT oid FROM pg_roles WHERE rolname = current_user) END" + ) + }; +} + +/// What relation `$2` of schema `$1` grants role `$3`, as (source, whether this connection can +/// take it back, privilege). +const RELATION_GRANTS: &str = concat!( + "SELECT ", + grant_source!("c.relowner"), + ", a.privilege_type + FROM pg_class c + JOIN pg_namespace n ON n.oid = c.relnamespace, + aclexplode(COALESCE(c.relacl, acldefault( + CASE c.relkind WHEN 'S' THEN 's' ELSE 'r' END::\"char\", c.relowner))) a + WHERE n.nspname = $1 AND c.relname = $2 + AND c.relkind = ANY(ARRAY['r','p','v','m','S','f']::\"char\"[]) + AND a.grantee = (SELECT oid FROM pg_roles WHERE rolname = $3)" +); + +/// The grants a revoke takes back, read from the catalog rather than from the request: one per +/// object and source that gave `pg_role` any of `privileges`. Refused when a source's grant is out +/// of this connection's reach (`grant_source!`, or for a default privilege a creating role it does +/// not act for): the revoke would leave that grant in place. +async fn read_revoked_grants( + client: &tokio_postgres::Client, + dbname: &str, + target: &AclTarget, + scope: GrantScope, + objects: &[AclObject], + privileges: &[String], + pg_role: &str, +) -> Result> { + let read_error = |e: tokio_postgres::Error| { + Error::internal_err(format!( + "Failed to read what the revoke takes back: {}", + pg_error_message(&e) + )) + }; + let wanted: Vec = privileges.iter().map(|p| p.to_uppercase()).collect(); + // (object, how it reads in a refusal, one row per source and privilege) + let mut read = Vec::new(); + match (scope, target) { + (scope, AclTarget::Schema { schema }) if scope.is_future() => { + let (objtype, plural) = match scope { + GrantScope::FutureTables => ("r", "tables"), + GrantScope::FutureSequences => ("S", "sequences"), + _ => ("f", "functions"), + }; + let database_wide = client + .query( + concat!( + "SELECT pg_get_userbyid(g.defaclrole), g.privilege_type FROM (", + database_wide_defaults!(), + ") g + WHERE g.defaclobjtype::text = $1 + AND g.grantee = (SELECT oid FROM pg_roles WHERE rolname = $2) + ORDER BY 1, 2" + ), + &[&objtype, &pg_role], + ) + .await + .map_err(read_error)?; + let mut still_granted: BTreeMap> = BTreeMap::new(); + for row in database_wide { + let privilege: String = row.get(1); + if wanted.contains(&privilege) { + still_granted.entry(row.get(0)).or_default().push(privilege); + } + } + if let Some((creator, taken)) = still_granted.into_iter().next() { + return Err(Error::BadRequest(format!( + "{} also receives {} on {plural} {} creates through a default privilege set \ + database-wide, which no default of schema {schema} takes back. Revoke it \ + database-wide first: ALTER DEFAULT PRIVILEGES FOR ROLE {} REVOKE {} ON {} \ + FROM {}", + role_name_of(pg_role), + taken.join(", "), + role_name_of(&creator), + quote_ident(&creator), + taken.join(", "), + default_objects_keyword(objtype), + quote_ident(pg_role) + ))); + } + let rows = client + .query( + "SELECT pg_get_userbyid(d.defaclrole), pg_has_role(d.defaclrole, 'USAGE'), + a.privilege_type + FROM pg_default_acl d + JOIN pg_namespace n ON n.oid = d.defaclnamespace, + aclexplode(d.defaclacl) a + WHERE n.nspname = $1 AND d.defaclobjtype::text = $2 + AND a.grantee = (SELECT oid FROM pg_roles WHERE rolname = $3)", + &[schema, &objtype, &pg_role], + ) + .await + .map_err(read_error)?; + read.push(( + None, + format!("{plural} created later in schema {schema}"), + rows, + )); + } + (GrantScope::Target, _) if objects.is_empty() => { + let rows = match target { + AclTarget::Database => { + client + .query( + concat!( + "SELECT ", + grant_source!("d.datdba"), + ", a.privilege_type + FROM pg_database d, + aclexplode(COALESCE(d.datacl, acldefault('d', d.datdba))) a + WHERE d.datname = current_database() + AND a.grantee = (SELECT oid FROM pg_roles WHERE rolname = $1)" + ), + &[&pg_role], + ) + .await + } + AclTarget::Schema { schema } => { + client + .query( + concat!( + "SELECT ", + grant_source!("n.nspowner"), + ", a.privilege_type + FROM pg_namespace n, + aclexplode(COALESCE(n.nspacl, acldefault('n', n.nspowner))) a + WHERE n.nspname = $1 + AND a.grantee = (SELECT oid FROM pg_roles WHERE rolname = $2)" + ), + &[schema, &pg_role], + ) + .await + } + AclTarget::Table { schema, table } => { + client + .query(RELATION_GRANTS, &[schema, table, &pg_role]) + .await + } + } + .map_err(read_error)?; + read.push((None, target.label(dbname), rows)); + } + (GrantScope::Target, AclTarget::Schema { schema }) => { + for object in objects { + let rows = match object_keyword(&object.kind)? { + "ROUTINE" => { + client + .query( + concat!( + "SELECT ", + grant_source!("p.proowner"), + ", a.privilege_type + FROM pg_proc p + JOIN pg_namespace n ON n.oid = p.pronamespace, + aclexplode(COALESCE(p.proacl, acldefault('f', p.proowner))) a + WHERE n.nspname = $1 AND p.proname = $2 + AND pg_get_function_identity_arguments(p.oid) = $3 + AND a.grantee = (SELECT oid FROM pg_roles WHERE rolname = $4)" + ), + &[ + schema, + &object.name, + &object.args.as_deref().unwrap_or(""), + &pg_role, + ], + ) + .await + } + _ => { + client + .query(RELATION_GRANTS, &[schema, &object.name, &pg_role]) + .await + } + } + .map_err(read_error)?; + read.push(( + Some(object.clone()), + format!("{} {schema}.{}", object.kind.to_lowercase(), object.name), + rows, + )); + } + } + // Every other scope and target is the planner's to refuse. + _ => {} + } + + let mut revoked = Vec::new(); + for (object, label, rows) in read { + let mut by_source: BTreeMap)> = BTreeMap::new(); + for row in rows { + let privilege: String = row.get(2); + if wanted.contains(&privilege) { + by_source + .entry(row.get(0)) + .or_insert_with(|| (row.get(1), vec![])) + .1 + .push(privilege); + } + } + for (source, (reachable, mut privileges)) in by_source { + if !reachable { + return Err(Error::BadRequest(format!( + "{} on {label} was granted to {} by {source}, and Postgres takes a grant \ + back only through the role that made it, which this data table's \ + connection cannot speak for here. Revoke it as {source}.", + privileges.join(", "), + role_name_of(pg_role), + ))); + } + privileges.sort(); + privileges.dedup(); + revoked.push(RevokedGrant { object: object.clone(), source, privileges }); + } + } + Ok(revoked) +} + +/// The keyword a `REVOKE ... ON` takes for one object, checked rather than interpolated: it lands +/// in SQL unquoted. +pub(crate) fn object_keyword(kind: &str) -> Result<&'static str> { + match kind.to_uppercase().as_str() { + "TABLE" | "VIEW" | "MATERIALIZED VIEW" | "FOREIGN TABLE" => Ok("TABLE"), + "SEQUENCE" => Ok("SEQUENCE"), + // `FUNCTION` names no procedure; `ROUTINE` names either. + "FUNCTION" | "PROCEDURE" | "ROUTINE" => Ok("ROUTINE"), + other => Err(Error::BadRequest(format!("Unknown object kind '{other}'"))), + } +} + +/// Every object of a schema, named the way the catalog names it. +async fn read_schema_objects( + client: &tokio_postgres::Client, + schema: &str, +) -> Result> { + let rows = client + .query( + "SELECT CASE c.relkind WHEN 'S' THEN 'SEQUENCE' ELSE 'TABLE' END, c.relname, NULL::text + FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace + WHERE n.nspname = $1 AND c.relkind = ANY(ARRAY['r','p','v','m','S','f']::\"char\"[]) + UNION ALL + SELECT CASE p.prokind WHEN 'p' THEN 'PROCEDURE' ELSE 'FUNCTION' END, p.proname, + pg_get_function_identity_arguments(p.oid) + FROM pg_proc p JOIN pg_namespace n ON n.oid = p.pronamespace + WHERE n.nspname = $1", + &[&schema], + ) + .await + .map_err(|e| { + Error::internal_err(format!( + "Failed to list the objects of schema '{schema}': {}", + pg_error_message(&e) + )) + })?; + Ok(rows + .into_iter() + .map(|row| AclObject { kind: row.get(0), name: row.get(1), args: row.get(2) }) + .collect()) +} + +/// Replace the objects a revoke names with the catalog's own entry for each. +/// +/// A routine is identified by its argument types, and those go into the statement as written — +/// there is no quoting for them — so the request may name an object but never spell one: what +/// reaches the SQL is read back from Postgres. An object that resolves to nothing is refused rather +/// than dropped, since a revoke that silently covers less than it says is worse than an error. +async fn resolve_acl_objects( + client: &tokio_postgres::Client, + target: &AclTarget, + objects: &[AclObject], +) -> Result> { + if objects.is_empty() { + return Ok(vec![]); + } + let Some(schema) = target.schema() else { + return Err(Error::BadRequest( + "A database has no objects of its own to revoke on".to_string(), + )); + }; + let known = read_schema_objects(client, schema).await?; + objects + .iter() + .map(|requested| { + let keyword = object_keyword(&requested.kind)?; + known + .iter() + .find(|k| { + k.name == requested.name + && k.args == requested.args + && object_keyword(&k.kind).is_ok_and(|k| k == keyword) + }) + .cloned() + .ok_or_else(|| { + Error::NotFound(format!( + "'{}' is not an object of schema '{schema}'", + requested.name + )) + }) + }) + .collect() +} + +async fn read_owner(client: &tokio_postgres::Client, target: &AclTarget) -> Result> { + let row = match target { + AclTarget::Database => { + client + .query_opt( + "SELECT pg_get_userbyid(datdba) FROM pg_database WHERE datname = current_database()", + &[], + ) + .await + } + AclTarget::Schema { schema } => { + client + .query_opt( + // `public` is owned by `pg_database_owner`, a placeholder role whose membership + // is whoever owns the database — naming it back would say nothing, so resolve + // it to that owner. + "SELECT pg_get_userbyid(owner) FROM ( + SELECT CASE WHEN n.nspowner = (SELECT oid FROM pg_roles WHERE rolname = 'pg_database_owner') + THEN (SELECT d.datdba FROM pg_database d WHERE d.datname = current_database()) + ELSE n.nspowner END AS owner + FROM pg_namespace n WHERE n.nspname = $1 + ) o", + &[schema], + ) + .await + } + AclTarget::Table { schema, table } => { + client + .query_opt( + "SELECT pg_get_userbyid(c.relowner) + FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace + WHERE n.nspname = $1 AND c.relname = $2", + &[schema, table], + ) + .await + } + } + .map_err(|e| Error::internal_err(format!("Failed to read the owner: {}", pg_error_message(&e))))?; + Ok(row.map(|row| row.get(0))) +} + +async fn read_children(client: &tokio_postgres::Client, target: &AclTarget) -> Result> { + let rows = match target { + AclTarget::Database => { + client + .query( + "SELECT nspname::text FROM pg_namespace + WHERE nspname <> 'information_schema' AND nspname NOT LIKE 'pg\\_%' + ORDER BY nspname", + &[], + ) + .await + } + AclTarget::Schema { schema } => { + client + .query( + "SELECT c.relname::text + FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace + WHERE n.nspname = $1 AND c.relkind = ANY(ARRAY['r','p']::\"char\"[]) + ORDER BY c.relname", + &[schema], + ) + .await + } + AclTarget::Table { .. } => return Ok(vec![]), + } + .map_err(|e| { + Error::internal_err(format!( + "Failed to list what the target holds: {}", + pg_error_message(&e) + )) + })?; + Ok(rows.into_iter().map(|row| row.get(0)).collect()) +} + +async fn get_datatable_acl( + authed: ApiAuthed, + Extension(db): Extension, + Path((w_id, datatable_name)): Path<(String, String)>, + Query(query): Query, +) -> JsonResult { + crate::datatable_acl_oss::ensure_datatable_acl_available()?; + let target: AclTarget = query.try_into()?; + let governing = resolve_governing_datatable(&db, &w_id, &datatable_name).await?; + ensure_reaches_governing_datatable(&db, &w_id, &datatable_name, &governing, &authed).await?; + ensure_instance(&governing)?; + let editable = ensure_governs_datatable(&db, &authed, &w_id, &governing) + .await + .is_ok(); + let roles = if editable { + role_names(&read_role_catalog(&db).await?) + } else { + vec![] + }; + + let (client, _notices, dbname) = connect_as_admin_unchecked(&db, &governing).await?; + let owner = read_owner(&client, &target) + .await? + .ok_or_else(|| Error::NotFound(format!("{} not found", target.label(&dbname))))?; + let grants = read_grants(&client, &target).await?; + let supports_maintain: bool = client + .query_one( + "SELECT current_setting('server_version_num')::int >= 170000", + &[], + ) + .await + .map_err(|e| { + Error::internal_err(format!( + "Failed to read the server version: {}", + pg_error_message(&e) + )) + })? + .get(0); + let children = read_children(&client, &target).await?; + + Ok(Json(DatatableAclInfo { + owner: role_name_of(&owner), + roles, + editable, + clone: governing.governor.is_some(), + supports_maintain, + dbname, + grants, + children, + })) +} + +async fn read_grants(client: &tokio_postgres::Client, target: &AclTarget) -> Result> { + // `aclexplode` turns an acl array into one row per (grantee, privilege); grantee 0 is PUBLIC, + // which has no name to resolve. A NULL acl is not "no access" but Postgres's built-in default — + // the owner holds everything and, on a routine, PUBLIC may EXECUTE — hence `acldefault`. The + // owner's own entries are left out: what it holds comes with ownership, which the owner shows, + // not with a grant a revoke here could take back. Each row ends with its source — the grantor, + // or a default privilege's creating role — and whether this connection can take back what it + // gave. + // Column-level grants (`pg_attribute.attacl`) are not supported yet: they are neither read here + // nor revocable from the editor. + let mut rows = match target { + AclTarget::Database => { + let mut out = client + .query( + concat!( + "SELECT CASE WHEN a.grantee = 0 THEN 'PUBLIC' ELSE pg_get_userbyid(a.grantee) END, + a.privilege_type, NULL::text, NULL::text, NULL::text, NULL::text, ", + grant_source!("d.datdba"), + " FROM pg_database d, aclexplode(COALESCE(d.datacl, acldefault('d', d.datdba))) a + WHERE d.datname = current_database() AND a.grantee <> d.datdba" + ), + &[], + ) + .await + .map_err(grant_read_error)?; + out.extend( + client + .query( + // Default privileges set database-wide reach what is created in every + // schema, so they are the database's to show rather than any schema's. + concat!( + "SELECT CASE WHEN g.grantee = 0 THEN 'PUBLIC' ELSE pg_get_userbyid(g.grantee) END, + g.privilege_type, NULL::text, + CASE g.defaclobjtype + WHEN 'r' THEN 'TABLES' WHEN 'S' THEN 'SEQUENCES' + WHEN 'f' THEN 'FUNCTIONS' WHEN 'n' THEN 'SCHEMAS' + ELSE 'TYPES' END, NULL::text, NULL::text, + pg_get_userbyid(g.defaclrole), pg_has_role(g.defaclrole, 'USAGE') + FROM (", + database_wide_defaults!(), + ") g" + ), + &[], + ) + .await + .map_err(grant_read_error)?, + ); + out + } + AclTarget::Schema { schema } => { + let mut out = client + .query( + concat!( + "SELECT CASE WHEN a.grantee = 0 THEN 'PUBLIC' ELSE pg_get_userbyid(a.grantee) END, + a.privilege_type, NULL::text, NULL::text, NULL::text, NULL::text, ", + grant_source!("n.nspowner"), + " FROM pg_namespace n, aclexplode(COALESCE(n.nspacl, acldefault('n', n.nspowner))) a + WHERE n.nspname = $1 AND a.grantee <> n.nspowner" + ), + &[schema], + ) + .await + .map_err(grant_read_error)?; + out.extend( + client + .query( + concat!( + "SELECT CASE WHEN a.grantee = 0 THEN 'PUBLIC' ELSE pg_get_userbyid(a.grantee) END, + a.privilege_type, c.relname, NULL::text, + CASE c.relkind WHEN 'S' THEN 'SEQUENCE' ELSE 'TABLE' END, + NULL::text, ", + grant_source!("c.relowner"), + " FROM pg_class c + JOIN pg_namespace n ON n.oid = c.relnamespace, + aclexplode(COALESCE(c.relacl, acldefault( + CASE c.relkind WHEN 'S' THEN 's' ELSE 'r' END::\"char\", c.relowner))) a + WHERE n.nspname = $1 + AND c.relkind = ANY(ARRAY['r','p','v','m','S','f']::\"char\"[]) + AND a.grantee <> c.relowner" + ), + &[schema], + ) + .await + .map_err(grant_read_error)?, + ); + out.extend( + client + .query( + // Routines carry their own acl in `pg_proc`; without this a grant made here + // would vanish on the next read and could never be revoked back. + concat!( + "SELECT CASE WHEN a.grantee = 0 THEN 'PUBLIC' ELSE pg_get_userbyid(a.grantee) END, + a.privilege_type, p.proname, NULL::text, + CASE p.prokind WHEN 'p' THEN 'PROCEDURE' ELSE 'FUNCTION' END, + pg_get_function_identity_arguments(p.oid), ", + grant_source!("p.proowner"), + " FROM pg_proc p + JOIN pg_namespace n ON n.oid = p.pronamespace, + aclexplode(COALESCE(p.proacl, acldefault('f', p.proowner))) a + WHERE n.nspname = $1 AND a.grantee <> p.proowner" + ), + &[schema], + ) + .await + .map_err(grant_read_error)?, + ); + out.extend( + client + .query( + // `USAGE` on a type is what lets a role use it in a column. Only a type the + // schema holds in its own right has an acl: an array, a row type and a + // multirange answer to their element, table or range. + concat!( + "SELECT CASE WHEN a.grantee = 0 THEN 'PUBLIC' ELSE pg_get_userbyid(a.grantee) END, + a.privilege_type, t.typname, NULL::text, 'TYPE', NULL::text, ", + grant_source!("t.typowner"), + " FROM pg_type t + JOIN pg_depend d ON d.classid = 'pg_type'::regclass AND d.objid = t.oid + AND d.refclassid = 'pg_namespace'::regclass AND d.deptype = 'n', + aclexplode(COALESCE(t.typacl, acldefault('T', t.typowner))) a + WHERE d.refobjid = (SELECT oid FROM pg_namespace WHERE nspname = $1) + AND a.grantee <> t.typowner + AND NOT EXISTS ( + SELECT 1 FROM pg_depend x + WHERE x.classid = 'pg_type'::regclass AND x.objid = t.oid + AND x.objsubid = 0 AND x.deptype = 'i')" + ), + &[schema], + ) + .await + .map_err(grant_read_error)?, + ); + out.extend( + client + .query( + // What a creating role set is taken back `FOR ROLE` that role, which only + // a role acting for it may do. + "SELECT CASE WHEN a.grantee = 0 THEN 'PUBLIC' ELSE pg_get_userbyid(a.grantee) END, + a.privilege_type, NULL::text, + CASE d.defaclobjtype + WHEN 'r' THEN 'TABLES' WHEN 'S' THEN 'SEQUENCES' + WHEN 'f' THEN 'FUNCTIONS' ELSE 'TYPES' END, NULL::text, NULL::text, + pg_get_userbyid(d.defaclrole), pg_has_role(d.defaclrole, 'USAGE') + FROM pg_default_acl d + JOIN pg_namespace n ON n.oid = d.defaclnamespace, + aclexplode(d.defaclacl) a + WHERE n.nspname = $1", + &[schema], + ) + .await + .map_err(grant_read_error)?, + ); + out + } + AclTarget::Table { schema, table } => client + .query( + concat!( + "SELECT CASE WHEN a.grantee = 0 THEN 'PUBLIC' ELSE pg_get_userbyid(a.grantee) END, + a.privilege_type, NULL::text, NULL::text, NULL::text, NULL::text, ", + grant_source!("c.relowner"), + " FROM pg_class c + JOIN pg_namespace n ON n.oid = c.relnamespace, + aclexplode(COALESCE(c.relacl, acldefault( + CASE c.relkind WHEN 'S' THEN 's' ELSE 'r' END::\"char\", c.relowner))) a + WHERE n.nspname = $1 AND c.relname = $2 AND a.grantee <> c.relowner" + ), + &[schema, table], + ) + .await + .map_err(grant_read_error)?, + }; + + // One row per privilege and source — and, for default privileges, per creating role. Fold them + // back into one entry per grantee and object that keeps every source and what each gave: a + // revoke takes a privilege back from every source that gave it. + let mut folded: BTreeMap< + ( + String, + Option<(String, String, Option)>, + Option, + ), + (Vec, BTreeMap)>), + > = BTreeMap::new(); + for row in rows.drain(..) { + let grantee: String = row.get(0); + let privilege: String = row.get(1); + let object: Option = row.get(2); + let future: Option = row.get(3); + let object_kind: Option = row.get(4); + let object_args: Option = row.get(5); + let source: String = row.get(6); + let reachable: bool = row.get(7); + let (privileges, sources) = folded + .entry(( + role_name_of(&grantee), + object.map(|name| { + ( + name, + object_kind.unwrap_or_else(|| "TABLE".to_string()), + object_args, + ) + }), + future, + )) + .or_default(); + sources + .entry(role_name_of(&source)) + .or_insert_with(|| (reachable, vec![])) + .1 + .push(privilege.clone()); + privileges.push(privilege); + } + Ok(folded + .into_iter() + .map(|((grantee, object, future), (mut privileges, sources))| { + privileges.sort(); + privileges.dedup(); + AclGrant { + grantee, + privileges, + object: object.map(|(name, kind, args)| AclObject { name, kind, args }), + future, + sources: sources + .into_iter() + .map(|(role, (reachable, mut privileges))| { + privileges.sort(); + privileges.dedup(); + AclSource { role, privileges, reachable } + }) + .collect(), + } + }) + .collect()) +} + +fn grant_read_error(e: tokio_postgres::Error) -> Error { + Error::internal_err(format!("Failed to read grants: {}", pg_error_message(&e))) +} + +/// Changing a data table's access is administering it. Checked in full before anything connects +/// with the instance's credentials. +async fn authorize_acl_change( + db: &DB, + authed: &ApiAuthed, + w_id: &str, + datatable_name: &str, +) -> Result { + let governing = resolve_governing_datatable(db, w_id, datatable_name).await?; + ensure_governs_datatable(db, authed, w_id, &governing).await?; + ensure_instance(&governing)?; + Ok(governing) +} + +static APPLY_SLOT: tokio::sync::Semaphore = tokio::sync::Semaphore::const_new(1); + +/// A role passes on only privileges it holds with grant option, and an instance database +/// provisioned before data table roles gave `custom_instance_user` none. Adds that option to its +/// database and `public` privileges, and nothing else: default privileges are left alone, since a +/// schema's change of owner is planned against them. Best-effort, as a grant it fails to enable is +/// refused when it runs. +async fn ensure_grant_options(client: &tokio_postgres::Client, db: &DB, dbname: &str) { + let held = client + .query_one( + "SELECT has_database_privilege(current_database(), 'CONNECT WITH GRANT OPTION') + AND has_database_privilege(current_database(), 'CREATE WITH GRANT OPTION') + AND (to_regnamespace('public') IS NULL + OR (has_schema_privilege('public', 'USAGE WITH GRANT OPTION') + AND has_schema_privilege('public', 'CREATE WITH GRANT OPTION')))", + &[], + ) + .await + .is_ok_and(|row| row.get::<_, bool>(0)); + if held { + return; + } + if let Err(e) = grant_options_as_server(db, dbname).await { + tracing::warn!("Could not enable grant options on '{dbname}': {e}"); + } +} + +/// Only the database's owner, the server's own Postgres user, can hand out an option it holds. +async fn grant_options_as_server(db: &DB, dbname: &str) -> Result<()> { + let server = PgDatabase::parse_uri(&windmill_common::get_database_url().await?.as_str().await)?; + let creds = PgDatabase { dbname: dbname.to_string(), ..server }; + let (client, connection) = creds.connect(Some(db)).await?; + let join_handle = tokio::spawn(async move { connection.await }); + let role = quote_ident(CUSTOM_INSTANCE_USER); + let result = client + .batch_execute(&format!( + "GRANT CONNECT, CREATE ON DATABASE {} TO {role} WITH GRANT OPTION; + DO $$ BEGIN + IF to_regnamespace('public') IS NOT NULL THEN + GRANT USAGE, CREATE ON SCHEMA public TO {role} WITH GRANT OPTION; + END IF; + END $$;", + quote_ident(dbname) + )) + .await; + drop(client); + windmill_common::shutdown_pg_connection(join_handle).await?; + result.map_err(|e| { + Error::internal_err(format!( + "Failed to grant options on '{dbname}': {}", + pg_error_message(&e) + )) + }) +} + +/// Whether the governing entry an apply was authorized on is still the one in the settings, read +/// under the lock: a save in between could have pointed it at another database or changed its roles. +fn entry_unchanged(governing: &GoverningDatatable, entry_now: Option) -> bool { + let Some(Ok(now)) = entry_now.map(serde_json::from_value::) else { + return false; + }; + match ( + serde_json::to_value(&now), + serde_json::to_value(&governing.datatable), + ) { + (Ok(now), Ok(authorized)) => now == authorized, + _ => false, + } +} + +/// Plan one change against the catalog and the database as they are now. +async fn build_plan( + client: &tokio_postgres::Client, + dbname: &str, + catalog: &DatatableRoleCatalog, + target: &AclTarget, + change: &AclChange, +) -> Result { + let role = change.role(); + // `admin` is the login the data table itself reaches Postgres through, and the one every + // change here runs as: a revoke that lands leaves nothing able to grant it back. + if matches!(change, AclChange::Revoke { .. }) && role == ADMIN_DATATABLE_ROLE { + return Err(Error::BadRequest(format!( + "'{ADMIN_DATATABLE_ROLE}' is how this data table reaches its database; \ + its own access is not revocable from here" + ))); + } + let pg_role = pg_role_of(role, catalog)?; + let change = match change { + AclChange::Revoke { role, privileges, scope, objects } => AclChange::Revoke { + role: role.clone(), + privileges: privileges.clone(), + scope: *scope, + objects: resolve_acl_objects(client, target, objects).await?, + }, + change => change.clone(), + }; + // Default privileges are recorded per creating role, and a schema's new owner is kept in reach + // of what the others create there, so both are written for every role there is. + let other_pg_roles = role_names(catalog) + .iter() + .map(|name| pg_role_of(name, catalog)) + .filter(|r| r.as_ref().map_or(true, |r| *r != pg_role)) + .collect::>>()?; + let (existing_objects, former_owner) = match (&change, target) { + (AclChange::SetOwner { .. }, AclTarget::Schema { schema }) => ( + read_owned_objects(client, schema).await?, + read_former_owner_defaults(client, schema, &pg_role).await?, + ), + _ => (vec![], None), + }; + let revoked_grants = match &change { + AclChange::Revoke { privileges, scope, objects, .. } => { + read_revoked_grants( + client, dbname, target, *scope, objects, privileges, &pg_role, + ) + .await? + } + _ => vec![], + }; + if matches!(change, AclChange::SetOwner { .. }) { + if let Some((object, owner)) = unmanaged_owner(client, target).await? { + return Err(Error::BadRequest(format!( + "{object} is owned by {owner}, which this data table's connection cannot act \ + for, so its owner cannot be changed from here" + ))); + } + } + let facts = CatalogFacts { other_pg_roles, existing_objects, former_owner, revoked_grants }; + let mut plan = + crate::datatable_acl_oss::plan_statements(target, &change, dbname, &pg_role, &facts)?; + if matches!(change, AclChange::SetOwner { .. }) { + if let Some(missing) = missing_owner_privilege(client, target, &pg_role).await? { + plan.warnings.push(format!( + "{role} does not have {missing}, which Postgres requires of a new owner, so this \ + will be refused. Grant it first." + )); + } + } + Ok(plan) +} + +/// Postgres only hands an object to a role that could have created it: a table to one with +/// `CREATE` on its schema, a schema to one with `CREATE` on the database. +async fn missing_owner_privilege( + client: &tokio_postgres::Client, + target: &AclTarget, + pg_role: &str, +) -> Result> { + let (row, missing) = match target { + AclTarget::Table { schema, .. } => ( + client + .query_one( + "SELECT has_schema_privilege($1::name, $2::text, 'CREATE')", + &[&pg_role, schema], + ) + .await, + format!("CREATE on schema {schema}"), + ), + AclTarget::Schema { .. } => ( + client + .query_one( + "SELECT has_database_privilege($1::name, current_database(), 'CREATE')", + &[&pg_role], + ) + .await, + "CREATE on the database".to_string(), + ), + AclTarget::Database => return Ok(None), + }; + let has: bool = row + .map_err(|e| { + Error::internal_err(format!( + "Failed to read the new owner's privileges: {}", + pg_error_message(&e) + )) + })? + .get(0); + Ok((!has).then_some(missing)) +} + +/// The first thing a change of owner would move that this connection cannot act for, with its +/// owner. Postgres lets only a member of the current owner move an object, and every change runs as +/// `custom_instance_user`, so an object it does not hold the owner of — `public`, owned by the +/// database's owner, above all — is refused here rather than at apply. Running as the instance's +/// own user instead would reach objects Windmill never created. +async fn unmanaged_owner( + client: &tokio_postgres::Client, + target: &AclTarget, +) -> Result> { + let read_error = |e: tokio_postgres::Error| { + Error::internal_err(format!( + "Failed to read who owns what the change moves: {}", + pg_error_message(&e) + )) + }; + match target { + AclTarget::Database => Ok(None), + AclTarget::Table { schema, table } => { + let row = client + .query_opt( + "SELECT pg_get_userbyid(c.relowner) + FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace + WHERE n.nspname = $1 AND c.relname = $2 AND NOT pg_has_role(c.relowner, 'USAGE')", + &[schema, table], + ) + .await + .map_err(read_error)?; + Ok(row.map(|row| (format!("{schema}.{table}"), row.get(0)))) + } + AclTarget::Schema { schema } => { + let row = client + .query_opt( + concat!( + "SELECT ord, identity, pg_get_userbyid(owner) FROM ( + SELECT 0 AS ord, NULL::text AS identity, n.nspowner AS owner + FROM pg_namespace n WHERE n.nspname = $1 + UNION ALL + SELECT 1, identity, owner FROM (", + schema_owned_objects!(), + ") m + ) o WHERE NOT pg_has_role(owner, 'USAGE') ORDER BY ord, identity LIMIT 1" + ), + &[schema], + ) + .await + .map_err(read_error)?; + Ok(row.map(|row| { + let label = row + .get::<_, Option>(1) + .unwrap_or_else(|| format!("schema {schema}")); + (label, row.get(2)) + })) + } + } +} + +async fn plan_datatable_acl( + authed: ApiAuthed, + Extension(db): Extension, + Path((w_id, datatable_name)): Path<(String, String)>, + Json(req): Json, +) -> JsonResult { + crate::datatable_acl_oss::ensure_datatable_acl_available()?; + let governing = authorize_acl_change(&db, &authed, &w_id, &datatable_name).await?; + let catalog = read_role_catalog(&db).await?; + let (client, _notices, dbname) = connect_as_admin_unchecked(&db, &governing).await?; + Ok(Json( + build_plan(&client, &dbname, &catalog, &req.target, &req.change).await?, + )) +} + +async fn apply_datatable_acl( + authed: ApiAuthed, + Extension(db): Extension, + Path((w_id, datatable_name)): Path<(String, String)>, + Json(req): Json, +) -> Result { + crate::datatable_acl_oss::ensure_datatable_acl_available()?; + let confirmed = req.statements.as_ref().ok_or_else(|| { + Error::BadRequest( + "An apply runs exactly the statements its plan showed; plan the change first" + .to_string(), + ) + })?; + // Everything that needs the pool happens before the locks: once `tx` holds them, a second pool + // connection could wait forever on a pool that concurrent applies, queued on the same locks, + // have exhausted. + let governing = authorize_acl_change(&db, &authed, &w_id, &datatable_name).await?; + // Applies queue on an instance-wide lock while each holds a direct connection to the instance's + // Postgres; unbounded, the queue alone could exhaust its connection limit. One at a time per + // server, and the ones waiting hold no connection at all. + let _slot = APPLY_SLOT + .acquire() + .await + .map_err(|e| Error::internal_err(format!("ACL apply slot closed: {e}")))?; + let (mut client, mut notices, dbname) = connect_as_admin_unchecked(&db, &governing).await?; + ensure_grant_options(&client, &db, &dbname).await; + + // Held until the change is committed: a role renamed or dropped meanwhile would change what + // the plan names, and a settings save could move the entry onto another database. Taken in the + // same order as the permissions save, so the two cannot deadlock. + let mut tx = db.begin().await?; + lock_role_catalog(&mut tx).await?; + let entry_now = sqlx::query_scalar::<_, Option>( + "SELECT datatable->'datatables'->$2 FROM workspace_settings WHERE workspace_id = $1 FOR UPDATE", + ) + .bind(&governing.workspace_id) + .bind(&governing.name) + .fetch_optional(&mut *tx) + .await? + .flatten(); + let catalog = read_role_catalog_tx(&mut tx).await?; + + let plan = build_plan(&client, &dbname, &catalog, &req.target, &req.change).await?; + if !entry_unchanged(&governing, entry_now) || &plan.statements != confirmed { + return Err(Error::BadRequest( + "The data table or its roles changed since this was planned, so it would no longer \ + run what was confirmed. Plan it again." + .to_string(), + )); + } + + // One transaction: a half-applied ownership transfer leaves one schema's objects owned by two + // different roles. + let pg_tx = client.transaction().await.map_err(|e| { + Error::internal_err(format!( + "Failed to open a transaction on the data table: {}", + pg_error_message(&e) + )) + })?; + execute_acl_statements(&pg_tx, &mut notices, &plan.statements).await?; + + // A schema's objects were listed before the transaction opened; one committed since would stay + // with its old owner. One created while this transaction is still open can still slip past, as + // Postgres has no lock that holds creation in a schema back. That is benign: it stays with its + // creator, like anything created there later, and moving that table fixes it. + if let (AclChange::SetOwner { role }, AclTarget::Schema { schema }) = (&req.change, &req.target) + { + let new_owner = pg_role_of(role, &catalog)?; + let straggler = pg_tx + .query_opt( + concat!( + "SELECT identity FROM (", + schema_owned_objects!(), + ") m WHERE owner <> (SELECT oid FROM pg_roles WHERE rolname = $2) + ORDER BY 1 LIMIT 1" + ), + &[schema, &new_owner], + ) + .await + .map_err(|e| { + Error::internal_err(format!( + "Failed to check what the schema holds: {}", + pg_error_message(&e) + )) + })?; + if let Some(row) = straggler { + return Err(Error::BadRequest(format!( + "{} appeared while this ran and would keep its old owner, so nothing was \ + applied. Plan it again.", + row.get::<_, String>(0) + ))); + } + } + + let target_label = req.target.label(&dbname); + audit_log( + &mut *tx, + &authed, + "workspaces.datatable_acl", + ActionKind::Update, + &governing.workspace_id, + Some(&governing.name), + Some( + [ + ("target", target_label.as_str()), + ("change", change_kind(&req.change)), + ("role", req.change.role()), + ] + .into(), + ), + ) + .await?; + pg_tx.commit().await.map_err(|e| { + Error::internal_err(format!( + "Failed to commit the changes: {}", + pg_error_message(&e) + )) + })?; + tx.commit().await?; + + windmill_common::feature_usage::log_feature_usage( + "datatable", + "acl_applied", + change_kind(&req.change), + ); + + Ok(format!("Updated access on {target_label}")) +} + +/// What kind of change, never what it named: the telemetry key and the audit's summary. +fn change_kind(change: &AclChange) -> &'static str { + match change { + AclChange::SetOwner { .. } => "owner", + AclChange::Grant { scope, .. } | AclChange::Revoke { scope, .. } if scope.is_future() => { + "default_privileges" + } + AclChange::Grant { .. } => "grant", + AclChange::Revoke { .. } => "revoke", + } +} + +#[cfg(test)] +mod tests { + use super::*; + use windmill_common::datatable_roles::InstanceDatatableRole; + + #[test] + fn a_role_is_its_own_postgres_role_except_admin() { + let catalog: DatatableRoleCatalog = BTreeMap::from([( + "role1".to_string(), + InstanceDatatableRole { name: "analytics".to_string(), enabled: true, pwd: None }, + )]); + assert_eq!( + pg_role_of("admin", &catalog).unwrap(), + "custom_instance_user" + ); + assert_eq!(pg_role_of("analytics", &catalog).unwrap(), "analytics"); + assert_eq!(role_name_of("custom_instance_user"), "admin"); + assert_eq!(role_name_of("analytics"), "analytics"); + // A catalog id, a name the catalog lacks, or the admin login spelled out never stands for + // some other role. + for unknown in ["role1", "operator", "custom_instance_user", "PUBLIC", ""] { + assert!( + matches!(pg_role_of(unknown, &catalog), Err(Error::BadRequest(_))), + "{unknown}" + ); + } + } + + /// A connection to the test's own database, the way the handlers reach a data table's. + async fn catalog_client(pool: &sqlx::PgPool) -> tokio_postgres::Client { + let mut config: tokio_postgres::Config = + std::env::var("DATABASE_URL").unwrap().parse().unwrap(); + config.dbname(pool.connect_options().get_database().unwrap()); + let (client, connection) = config.connect(tokio_postgres::NoTls).await.unwrap(); + tokio::spawn(connection); + client + } + + /// What a revoke takes back is read from the catalog, per object and source, and only the + /// privileges it asks for: the planner renders exactly this, and refuses an empty read. + #[sqlx::test(migrations = false)] + async fn a_revoke_reads_back_only_what_it_asks_for(pool: sqlx::PgPool) { + let client = catalog_client(&pool).await; + // A predefined role, so that nothing is granted outside the test's own database. + client + .batch_execute( + "CREATE SCHEMA granted; + CREATE TABLE granted.g (id int); + GRANT SELECT, INSERT ON granted.g TO pg_read_all_data;", + ) + .await + .unwrap(); + let target = AclTarget::Table { schema: "granted".to_string(), table: "g".to_string() }; + let revoked = read_revoked_grants( + &client, + "db", + &target, + GrantScope::Target, + &[], + &["select".to_string()], + "pg_read_all_data", + ) + .await + .unwrap(); + assert_eq!(revoked.len(), 1, "{revoked:?}"); + assert_eq!(revoked[0].object, None); + assert_eq!(revoked[0].privileges, ["SELECT"]); + let held_none = read_revoked_grants( + &client, + "db", + &target, + GrantScope::Target, + &[], + &["update".to_string()], + "pg_read_all_data", + ) + .await + .unwrap(); + assert!(held_none.is_empty(), "{held_none:?}"); + // Each source says what it gave: that, and not the whole row, is what a revoke of some of + // its privileges is held back by. + let grants = read_grants( + &client, + &AclTarget::Schema { schema: "granted".to_string() }, + ) + .await + .unwrap(); + let on_g = grants + .iter() + .find(|g| { + g.grantee == "pg_read_all_data" && g.object.as_ref().is_some_and(|o| o.name == "g") + }) + .unwrap(); + assert_eq!(on_g.sources.len(), 1, "{:?}", on_g.sources); + assert_eq!(on_g.sources[0].privileges, ["INSERT", "SELECT"]); + assert!(on_g.sources[0].reachable); + } + + /// A default privilege set database-wide applies in every schema on top of the schema's own, + /// which cannot take it back: the database shows it, and neither a schema's revoke of it nor a + /// change of owner away from its grantee may go ahead as if it were gone. + #[sqlx::test(migrations = false)] + async fn a_schema_cannot_take_back_a_database_wide_default(pool: sqlx::PgPool) { + let client = catalog_client(&pool).await; + client + .batch_execute( + "CREATE SCHEMA owned AUTHORIZATION pg_read_all_data; + ALTER DEFAULT PRIVILEGES GRANT SELECT ON TABLES TO pg_read_all_data; + ALTER DEFAULT PRIVILEGES IN SCHEMA owned + GRANT SELECT, INSERT ON TABLES TO pg_read_all_data; + CREATE SCHEMA creators AUTHORIZATION pg_monitor; + ALTER DEFAULT PRIVILEGES FOR ROLE pg_monitor + GRANT SELECT ON TABLES TO pg_read_all_stats;", + ) + .await + .unwrap(); + let creator: String = client + .query_one("SELECT current_user::text", &[]) + .await + .unwrap() + .get(0); + + let grants = read_grants(&client, &AclTarget::Database).await.unwrap(); + let database_wide = grants + .iter() + .find(|g| g.grantee == "pg_read_all_data" && g.future.as_deref() == Some("TABLES")) + .unwrap_or_else(|| panic!("{grants:?}")); + assert_eq!(database_wide.privileges, ["SELECT"]); + // A database-wide entry also holds its creator's own privileges, which come with creating + // and are no grant: neither a row nor a refusal may stem from them. + assert!( + !grants + .iter() + .any(|g| g.future.is_some() && (g.grantee == creator || g.grantee == "pg_monitor")), + "{grants:?}" + ); + assert_eq!( + database_wide.sources.len(), + 1, + "{:?}", + database_wide.sources + ); + assert_eq!(database_wide.sources[0].role, creator); + + let schema = AclTarget::Schema { schema: "owned".to_string() }; + let still_granted = read_revoked_grants( + &client, + "db", + &schema, + GrantScope::FutureTables, + &[], + &["select".to_string()], + "pg_read_all_data", + ) + .await; + assert!( + matches!(&still_granted, Err(Error::BadRequest(m)) if m.contains("database-wide")), + "{still_granted:?}" + ); + let schema_only = read_revoked_grants( + &client, + "db", + &schema, + GrantScope::FutureTables, + &[], + &["insert".to_string()], + "pg_read_all_data", + ) + .await + .unwrap(); + assert_eq!(schema_only.len(), 1, "{schema_only:?}"); + assert_eq!(schema_only[0].privileges, ["INSERT"]); + + let moved = read_former_owner_defaults(&client, "owned", "pg_write_all_data").await; + assert!( + matches!(&moved, Err(Error::BadRequest(m)) if m.contains("database-wide")), + "{moved:?}" + ); + let moved_from_creator = + read_former_owner_defaults(&client, "creators", "pg_write_all_data").await; + assert!( + matches!(moved_from_creator, Ok(None)), + "{moved_from_creator:?}" + ); + } + + /// Defaults on types go with a schema's owner like the other kinds: once a database-wide + /// default takes PUBLIC's USAGE on types away, they are all that reaches a new type. + #[sqlx::test(migrations = false)] + async fn a_schemas_former_owner_defaults_include_types(pool: sqlx::PgPool) { + let client = catalog_client(&pool).await; + client + .batch_execute( + "CREATE SCHEMA typed AUTHORIZATION pg_read_all_data; + ALTER DEFAULT PRIVILEGES IN SCHEMA typed GRANT USAGE ON TYPES TO pg_read_all_data; + ALTER DEFAULT PRIVILEGES FOR ROLE pg_read_all_data IN SCHEMA typed + GRANT SELECT ON TABLES TO pg_read_all_data;", + ) + .await + .unwrap(); + let creator: String = client + .query_one("SELECT current_user::text", &[]) + .await + .unwrap() + .get(0); + let former = read_former_owner_defaults(&client, "typed", "pg_write_all_data") + .await + .unwrap(); + assert_eq!( + former, + Some(FormerOwnerDefaults { + pg_role: "pg_read_all_data".to_string(), + defaults: vec![(creator, "TYPES")], + }) + ); + } + + /// A kind of object the list misses stays with its old owner while the schema changes hands, + /// which only a real catalog shows. + #[sqlx::test(migrations = false)] + async fn a_schemas_owner_change_takes_every_object_in_it(pool: sqlx::PgPool) { + let client = catalog_client(&pool).await; + client + .batch_execute( + "CREATE SCHEMA moved; + CREATE TABLE moved.t (id serial PRIMARY KEY, a int, b int); + CREATE STATISTICS moved.st ON a, b FROM moved.t; + CREATE TABLE moved.pt (id int) PARTITION BY RANGE (id); + CREATE TABLE moved.pt1 PARTITION OF moved.pt FOR VALUES FROM (0) TO (10); + CREATE TYPE moved.r AS RANGE (subtype = float8); + CREATE TYPE moved.pair AS (a int, b int); + CREATE TYPE moved.mood AS ENUM ('ok'); + CREATE DOMAIN moved.tags AS text[]; + CREATE COLLATION moved.coll (provider = libc, locale = 'C');", + ) + .await + .unwrap(); + let owned = read_owned_objects(&client, "moved").await.unwrap(); + // Not the serial's sequence, the range's constructors and multirange, or any array or row + // type: each follows the object it belongs to. + assert_eq!( + owned + .iter() + .map(|o| (o.keyword, o.identity.as_str())) + .collect::>(), + [ + ("COLLATION", "moved.coll"), + ("STATISTICS", "moved.st"), + ("TABLE", "moved.pt"), + ("TABLE", "moved.pt1"), + ("TABLE", "moved.t"), + ("TYPE", "moved.mood"), + ("TYPE", "moved.pair"), + ("TYPE", "moved.r"), + ("TYPE", "moved.tags"), + ] + ); + } +} diff --git a/backend/windmill-api-workspaces/src/datatable_acl_oss.rs b/backend/windmill-api-workspaces/src/datatable_acl_oss.rs new file mode 100644 index 0000000000..59eb8cdbe0 --- /dev/null +++ b/backend/windmill-api-workspaces/src/datatable_acl_oss.rs @@ -0,0 +1,44 @@ +/* + * Author: Ruben Fiszel + * Copyright: Windmill Labs, Inc 2022 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + +//! Where the ACL planner comes from: the enterprise one, or a refusal. +//! +//! Data table roles are an Enterprise Edition feature, and so is everything here — reading who +//! owns what included. `private` alone is not that edition — community builds carry it — so the +//! planner is behind `enterprise` as well. + +#[cfg(all(feature = "private", feature = "enterprise"))] +pub(crate) use crate::datatable_acl_ee::plan_statements; + +#[cfg(all(feature = "private", feature = "enterprise"))] +pub(crate) fn ensure_datatable_acl_available() -> windmill_common::error::Result<()> { + Ok(()) +} + +#[cfg(not(all(feature = "private", feature = "enterprise")))] +use { + crate::datatable_acl::{AclChange, AclPlan, AclTarget, CatalogFacts}, + windmill_common::{datatable_roles_oss::datatable_roles_unavailable, error::Result}, +}; + +/// Checked first by every ACL route, before anything is read or connected to. +#[cfg(not(all(feature = "private", feature = "enterprise")))] +pub(crate) fn ensure_datatable_acl_available() -> Result<()> { + Err(datatable_roles_unavailable()) +} + +#[cfg(not(all(feature = "private", feature = "enterprise")))] +pub(crate) fn plan_statements( + _target: &AclTarget, + _change: &AclChange, + _dbname: &str, + _pg_role: &str, + _facts: &CatalogFacts, +) -> Result { + Err(datatable_roles_unavailable()) +} diff --git a/backend/windmill-api-workspaces/src/datatable_clone.rs b/backend/windmill-api-workspaces/src/datatable_clone.rs new file mode 100644 index 0000000000..a57a2216e5 --- /dev/null +++ b/backend/windmill-api-workspaces/src/datatable_clone.rs @@ -0,0 +1,566 @@ +/* + * Author: Ruben Fiszel + * Copyright: Windmill Labs, Inc 2022 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + +//! Copying a data table's database for the fork being created. +//! +//! The fork request makes its copies before it writes the fork: each is created, filled and — for a +//! data table under roles — given the source's owners and grants. What each copy was made from +//! stays in the request ([`MadeCopy`]), so the fork's transaction checks it against the source as it +//! is then, and a fork that fails drops the instance copies it made ([`drop_copies_after`]) and names +//! the others, which live on servers of the workspace's own. + +use std::collections::BTreeSet; + +use windmill_api_auth::ApiAuthed; +use windmill_common::datatable_roles::{lock_role_catalog, read_role_catalog_tx}; +use windmill_common::error::{pg_error_message, Error, Result}; +use windmill_common::utils::require_admin; +use windmill_common::worker::CLOUD_HOSTED; +use windmill_common::workspaces::{ + get_datatable_resource_from_db_unchecked, DataTableDatabase, DataTableForkBehavior, + GoverningDatatable, +}; +use windmill_common::{PgDatabase, DB}; + +use crate::datatable_acl::connect_with_notices; +use crate::datatable_permissions::ensure_reaches_governing_datatable; +use crate::workspaces::{ + create_database_on_server, ensure_datatable_is_clonable, pg_dump_database, pg_import_dump, + DumpFile, PgDumpOptions, +}; + +/// A database this request created and filled for one data table of the fork. +pub(crate) struct MadeCopy { + /// The data table's name, in the parent and in the fork. + pub(crate) name: String, + pub(crate) dbname: String, + pub(crate) behavior: DataTableForkBehavior, + /// The database the source resolved to when it was copied. + pub(crate) source_database: DataTableDatabase, + /// Whether the source's owners and grants were replayed into the copy: it was under roles. + pub(crate) replayed: bool, + /// For a resource-backed copy, the resource and variables its connection was resolved from. + pub(crate) connection: Option, +} + +/// What a resource-backed data table's connection is resolved from: its resource, and every +/// resource and variable that one references, as stored — and, for a secret kept in an external +/// backend, the value that backend holds. The connection is resolved from the snapshot itself +/// ([`ConnectionSnapshot::resolve`]), so it is exactly the one these rows describe. +#[derive(PartialEq)] +pub(crate) struct ConnectionSnapshot { + root: String, + resources: std::collections::BTreeMap>, + /// Stored value and whether it is secret. + variables: std::collections::BTreeMap>, + external_secrets: std::collections::BTreeMap, +} + +/// Read the [`ConnectionSnapshot`] of resource `resource_path` in workspace `w_id`. +/// +/// Authorization: none, and it holds secret values. Callers MUST have authorized using that +/// resource, and never disclose the snapshot or what it resolves to. +pub(crate) async fn connection_snapshot( + db: &DB, + conn: &mut sqlx::PgConnection, + w_id: &str, + resource_path: &str, +) -> Result { + let root = resource_path.trim_start_matches("$res:").to_string(); + let mut snapshot = ConnectionSnapshot { + root: root.clone(), + resources: Default::default(), + variables: Default::default(), + external_secrets: Default::default(), + }; + let mut pending = vec![root]; + while let Some(path) = pending.pop() { + if snapshot.resources.contains_key(&path) { + continue; + } + let value: Option = + sqlx::query_scalar("SELECT value FROM resource WHERE workspace_id = $1 AND path = $2") + .bind(w_id) + .bind(&path) + .fetch_optional(&mut *conn) + .await? + .flatten(); + let mut strings = vec![]; + collect_strings(value.as_ref(), &mut strings); + for reference in strings { + if let Some(var) = reference.strip_prefix("$var:") { + if snapshot.variables.contains_key(var) { + continue; + } + let row: Option<(String, bool)> = sqlx::query_as( + "SELECT value, is_secret FROM variable WHERE workspace_id = $1 AND path = $2", + ) + .bind(w_id) + .bind(var) + .fetch_optional(&mut *conn) + .await?; + if let Some((stored, true)) = &row { + if windmill_common::secret_backend::is_external_stored_value(stored) { + let secret = windmill_common::secret_backend::get_secret_value( + db, w_id, var, stored, + ) + .await?; + snapshot.external_secrets.insert(var.to_string(), secret); + } + } + snapshot.variables.insert(var.to_string(), row); + } else if let Some(res) = reference.strip_prefix("$res:") { + pending.push(res.to_string()); + } + } + snapshot.resources.insert(path, value); + } + Ok(snapshot) +} + +impl ConnectionSnapshot { + /// The connection these rows resolve to, as the data table's own resolution would: references + /// substituted, secrets decrypted. + pub(crate) async fn resolve(&self, db: &DB, w_id: &str) -> Result { + let root = self.resource(&self.root)?; + self.substitute(db, w_id, root, 0).await + } + + /// Whether the resource itself holds the connection's fields, which the fork repoints by setting + /// its `dbname`, rather than being a reference to another resource. + pub(crate) fn root_holds_connection(&self) -> bool { + self.resource(&self.root).is_ok_and(|v| v.is_object()) + } + + fn resource(&self, path: &str) -> Result<&serde_json::Value> { + self.resources + .get(path) + .and_then(|v| v.as_ref()) + .ok_or_else(|| Error::NotFound(format!("resource {path} does not exist"))) + } + + fn substitute<'a>( + &'a self, + db: &'a DB, + w_id: &'a str, + value: &'a serde_json::Value, + depth: usize, + ) -> std::pin::Pin> + Send + 'a>> + { + Box::pin(async move { + if depth > 32 { + return Err(Error::BadRequest( + "resource references nest too deeply".to_string(), + )); + } + Ok(match value { + serde_json::Value::Object(map) => { + let mut out = serde_json::Map::new(); + for (key, val) in map { + out.insert(key.clone(), self.substitute(db, w_id, val, depth).await?); + } + serde_json::Value::Object(out) + } + serde_json::Value::Array(items) => { + let mut out = vec![]; + for val in items { + out.push(self.substitute(db, w_id, val, depth).await?); + } + serde_json::Value::Array(out) + } + serde_json::Value::String(s) if s.starts_with("$res:") => { + let path = &s[5..]; + self.substitute(db, w_id, self.resource(path)?, depth + 1) + .await? + } + serde_json::Value::String(s) if s.starts_with("$var:") => { + let path = &s[5..]; + let (stored, secret) = self + .variables + .get(path) + .and_then(|v| v.as_ref()) + .ok_or_else(|| { + Error::NotFound(format!("variable {path} does not exist")) + })?; + serde_json::Value::String(match (secret, self.external_secrets.get(path)) { + (false, _) => stored.clone(), + (true, Some(external)) => external.clone(), + (true, None) => windmill_common::variables::decrypt( + &windmill_common::variables::build_crypt(db, w_id).await?, + stored.clone(), + ) + .map_err(|e| { + Error::internal_err(format!("Error decrypting variable {s}: {e}")) + })?, + }) + } + other => other.clone(), + }) + }) + } +} + +fn collect_strings<'a>(value: Option<&'a serde_json::Value>, out: &mut Vec<&'a str>) { + match value { + Some(serde_json::Value::String(s)) => out.push(s), + Some(serde_json::Value::Array(items)) => { + items.iter().for_each(|v| collect_strings(Some(v), out)) + } + Some(serde_json::Value::Object(map)) => { + map.values().for_each(|v| collect_strings(Some(v), out)) + } + _ => {} + } +} + +/// What a data table of the fork should be copied as. +pub(crate) struct CopyRequest<'a> { + pub(crate) name: &'a str, + pub(crate) dbname: &'a str, + pub(crate) behavior: DataTableForkBehavior, +} + +/// Check that `authed` may copy each data table of `parent_w_id` as asked, before anything is +/// created. +/// +/// Who may copy is what it was before data table roles — anyone for the schema, an admin of the +/// workspace for the rows — narrowed under roles to whoever may connect as one of them: even the +/// schema alone lists every table, which under roles only role holders can read in the parent. +pub(crate) async fn authorize_copies( + db: &DB, + authed: &ApiAuthed, + parent_w_id: &str, + requests: &[CopyRequest<'_>], +) -> Result<()> { + for request in requests { + match request.behavior { + DataTableForkBehavior::KeepOriginal => { + return Err(Error::BadRequest(format!( + "Data table '{}' is kept, which copies nothing", + request.name + ))) + } + DataTableForkBehavior::SchemaOnly => {} + DataTableForkBehavior::SchemaAndData => { + require_admin(authed.is_admin, &authed.username)?; + if *CLOUD_HOSTED { + return Err(Error::BadRequest( + "Cloning schema and data is not available on cloud".to_string(), + )); + } + } + } + windmill_common::validate_dbname(request.dbname)?; + if !request.dbname.starts_with("wm_fork_") { + return Err(Error::BadRequest(format!( + "Forked datatable database name '{}' must start with 'wm_fork_'", + request.dbname + ))); + } + let governing = ensure_datatable_is_clonable(db, parent_w_id, request.name).await?; + ensure_reaches_governing_datatable(db, parent_w_id, request.name, &governing, authed) + .await?; + if governing.datatable.permissions.is_some() { + crate::datatable_replay_oss::ensure_replay()?; + } + } + Ok(()) +} + +/// Make every copy in `requests`, in order, once [`authorize_copies`] allows them all. When one +/// fails, the copies already made are dropped before the error is returned. +pub(crate) async fn make_copies( + db: &DB, + authed: &ApiAuthed, + parent_w_id: &str, + requests: &[CopyRequest<'_>], +) -> Result> { + authorize_copies(db, authed, parent_w_id, requests).await?; + let mut copies = Vec::with_capacity(requests.len()); + for request in requests { + match make_copy(db, authed, parent_w_id, request).await { + Ok(copy) => copies.push(copy), + Err(e) => return Err(drop_copies_after(db, copies, e).await), + } + } + Ok(copies) +} + +/// Drop every copy, returning `error` — with what could not be dropped appended, since nothing +/// else will name those databases again. +pub(crate) async fn drop_copies_after(db: &DB, copies: Vec, error: Error) -> Error { + let mut stranded = Vec::new(); + for copy in copies { + if let Err(e) = drop_copy(db, ©.source_database, ©.dbname).await { + tracing::error!("Could not drop '{}' after a failed fork: {e}", copy.dbname); + stranded.push(format!("'{}' ({e})", copy.dbname)); + } + } + if stranded.is_empty() { + error + } else { + Error::ExecutionErr(format!( + "{error}. These databases created for the fork could not be dropped: {}", + stranded.join(", ") + )) + } +} + +async fn drop_copy(db: &DB, source_database: &DataTableDatabase, dbname: &str) -> Result<()> { + if source_database.resource_type + == windmill_common::workspaces::DataTableCatalogResourceType::Instance + { + match windmill_common::drop_unused_instance_database(db, dbname).await? { + windmill_common::Cleanup::Dropped => Ok(()), + windmill_common::Cleanup::InUse(users) => Err(Error::BadRequest(format!( + "kept, since workspaces {} now use it", + users.join(", ") + ))), + windmill_common::Cleanup::Waiter(pid) => Err(Error::BadRequest(format!( + "kept, since a request (pid {pid}) is still waiting to name it" + ))), + } + } else { + // On a server of the workspace's own, where a resource edited meanwhile can already name it + // and nothing locks such an edit: dropping it could take someone's data. + Err(Error::BadRequest( + "kept on its PostgreSQL server, where it may already be in use; drop it there once it \ + is not, before forking the same data table under this id again" + .to_string(), + )) + } +} + +async fn make_copy( + db: &DB, + authed: &ApiAuthed, + parent_w_id: &str, + request: &CopyRequest<'_>, +) -> Result { + let governing = ensure_datatable_is_clonable(db, parent_w_id, request.name).await?; + let source_database = governing.datatable.database.clone().ok_or_else(|| { + Error::internal_err(format!( + "Data table '{}' resolves to an entry that owns no database", + request.name + )) + })?; + let is_instance = governing.is_instance(); + // Resolved from the snapshot, not read again: the connection the copy is made on is then + // exactly what these rows describe, which the fork's own clone of them is checked against. + let (server, connection): (PgDatabase, Option) = if is_instance { + let server = serde_json::from_value( + get_datatable_resource_from_db_unchecked(db, parent_w_id, request.name).await?, + ) + .map_err(|e| Error::internal_err(format!("Failed to parse database credentials: {e}")))?; + (server, None) + } else { + let snapshot = connection_snapshot( + db, + &mut *db.acquire().await?, + &governing.workspace_id, + &source_database.resource_path, + ) + .await?; + if !snapshot.root_holds_connection() { + return Err(Error::BadRequest(format!( + "Data table '{}' uses resource '{}', which only refers to another resource: the \ + fork's copy of it could not be pointed at the copied database. Point the data \ + table at the resource holding the connection, then fork again.", + request.name, source_database.resource_path + ))); + } + let server = serde_json::from_value(snapshot.resolve(db, &governing.workspace_id).await?) + .map_err(|e| { + Error::internal_err(format!("Failed to parse database credentials: {e}")) + })?; + (server, Some(snapshot)) + }; + + // Dumped before the database exists, so a source that cannot be read leaves nothing behind. + // Ownership never carries over: the restore runs as the target's connection user. Grants do, + // except on the instance, where the replay below is what reproduces them. + let dump = pg_dump_database( + &server, + PgDumpOptions { + schema_only: request.behavior == DataTableForkBehavior::SchemaOnly, + no_owner: true, + no_acl: is_instance, + ..Default::default() + }, + ) + .await?; + + if is_instance { + windmill_common::create_custom_instance_database(db, request.dbname, "datatable").await?; + } else { + create_database_on_server(db, &server, request.dbname).await?; + } + + let target = PgDatabase { dbname: request.dbname.to_string(), ..server.clone() }; + let filled = fill( + db, + authed, + parent_w_id, + request.name, + &governing, + &server, + &target, + &dump, + ) + .await; + match filled { + Ok(replayed) => Ok(MadeCopy { + name: request.name.to_string(), + dbname: request.dbname.to_string(), + behavior: request.behavior, + source_database, + replayed, + connection, + }), + Err(e) => { + if let Err(drop_err) = drop_copy(db, &source_database, request.dbname).await { + tracing::error!( + "Could not drop '{}' after a failed copy of data table '{}': {drop_err}", + request.dbname, + request.name + ); + return Err(Error::ExecutionErr(format!( + "{e}. The database '{}' created for the copy could not be dropped: {drop_err}", + request.dbname + ))); + } + Err(e) + } + } +} + +/// Restore the dump into the new database and, for a data table under roles, replay the source's +/// owners and grants into it. Returns whether it replayed. +#[allow(clippy::too_many_arguments)] +async fn fill( + db: &DB, + authed: &ApiAuthed, + parent_w_id: &str, + name: &str, + governing: &GoverningDatatable, + source: &PgDatabase, + target: &PgDatabase, + dump: &DumpFile, +) -> Result { + pg_import_dump(target, dump).await?; + + // Held until the replay commits: a role renamed or dropped meanwhile would change what the + // replay names, and a settings save could move the source onto another database or put it under + // roles. Taken in the same order as the permissions save and the ACL apply. + // On a connection of its own: the checks below take theirs from the pool, and a lock holder + // drawn from the pool too could leave a small one with nothing to give them. + let database_url = windmill_common::get_database_url().await?; + let mut lock_holder = ::connect_with( + &database_url.connect_options().await?, + ) + .await + .map_err(|e| Error::internal_err(format!("Failed to connect to the database: {e}")))?; + let mut tx = sqlx::Connection::begin(&mut lock_holder).await?; + lock_role_catalog(&mut tx).await?; + lock_settings_rows(&mut tx, parent_w_id, name).await?; + + // Everything so far was decided before the locks. + let now = ensure_datatable_is_clonable(db, parent_w_id, name).await?; + ensure_reaches_governing_datatable(db, parent_w_id, name, &now, authed).await?; + if !same_database(&now.datatable.database, &governing.datatable.database) { + return Err(Error::BadRequest(format!( + "Data table '{name}' moved to another database while it was being copied; fork again" + ))); + } + + let replayed = now.datatable.permissions.is_some(); + if replayed { + crate::datatable_replay_oss::ensure_replay()?; + let catalog = read_role_catalog_tx(&mut tx).await?; + // The replay leaves `CONNECT` to the catalog, and creating the database only tried to set + // it: a copy `PUBLIC` could still connect to would admit logins the source turns away. + windmill_common::datatable_roles::converge_connect_grants_with( + db, + &target.dbname, + &catalog, + ) + .await?; + let catalog_roles: BTreeSet = catalog.values().map(|r| r.name.clone()).collect(); + let (source, _source_notices) = connect_with_notices(db, source).await?; + let (mut target, mut notices) = connect_with_notices(db, target).await?; + // One transaction on the copy: a replay that stops halfway leaves objects owned by one role + // and granted as another. + let pg_tx = target.transaction().await.map_err(|e| { + Error::internal_err(format!( + "Failed to open a transaction on the copy: {}", + pg_error_message(&e) + )) + })?; + crate::datatable_replay_oss::replay_owners_and_grants( + &source, + &pg_tx, + &mut notices, + &catalog_roles, + ) + .await?; + pg_tx.commit().await.map_err(|e| { + Error::internal_err(format!( + "Failed to commit the replayed grants: {}", + pg_error_message(&e) + )) + })?; + } + tx.commit().await?; + Ok(replayed) +} + +/// Lock every settings row the resolution of data table `name` of `start_w_id` passes through, in +/// the order it passes them — each pointer, the entry that owns the database, and each workspace its +/// roles come from — and return them. Deleting or repointing any of them would leave a copy made +/// for the resolution answering for another one. +pub(crate) async fn lock_settings_rows( + tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, + start_w_id: &str, + name: &str, +) -> Result> { + let mut path: Vec<(String, String)> = vec![]; + let (mut w_id, mut datatable) = (start_w_id.to_string(), name.to_string()); + loop { + if path.contains(&(w_id.clone(), datatable.clone())) || path.len() > 32 { + return Err(Error::BadRequest(format!( + "Data table '{name}' of workspace '{start_w_id}' resolves through a cycle" + ))); + } + let entry: Option = sqlx::query_scalar( + "SELECT datatable->'datatables'->$2 FROM workspace_settings + WHERE workspace_id = $1 FOR UPDATE", + ) + .bind(&w_id) + .bind(&datatable) + .fetch_optional(&mut **tx) + .await? + .flatten(); + path.push((w_id.clone(), datatable.clone())); + let entry: Option = + entry.and_then(|e| serde_json::from_value(e).ok()); + match entry.and_then(|e| e.reference.or(e.governed_by)) { + Some(next) => (w_id, datatable) = (next.workspace_id, next.datatable), + None => break, + } + } + Ok(path.into_iter().map(|(w_id, _)| w_id).collect()) +} + +pub(crate) fn same_database(a: &Option, b: &Option) -> bool { + match (a, b) { + (Some(a), Some(b)) => { + a.resource_type == b.resource_type && a.resource_path == b.resource_path + } + _ => false, + } +} diff --git a/backend/windmill-api-workspaces/src/datatable_migrations.rs b/backend/windmill-api-workspaces/src/datatable_migrations.rs index fa433b2cbc..df1fac203b 100644 --- a/backend/windmill-api-workspaces/src/datatable_migrations.rs +++ b/backend/windmill-api-workspaces/src/datatable_migrations.rs @@ -107,26 +107,63 @@ async fn ensure_migration_role_allowed( datatable_name: &str, authed: &ApiAuthed, sql: &str, - timestamp: i64, - name: &str, + migration: &str, ) -> Result<()> { - let context = format!("Migration {timestamp} ({name})"); let access = DatatableAccess::Authed(authed.to_authed_ref()); match SqlAnnotations::datatable_role(sql)? { Some(role) => { - ensure_can_use_datatable_role(db, w_id, datatable_name, Some(&role), &access, &context) + ensure_can_use_datatable_role(db, w_id, datatable_name, Some(&role), &access, migration) .await } None => ensure_datatable_admin_access(db, w_id, datatable_name, &access) .await .map_err(|e| { Error::NotAuthorized(format!( - "{context} declares no role, so it would run as admin. {e}" + "{migration} declares no role, so it would run as admin. {e}" )) }), } } +/// Refuse writing or deleting a migration definition unless this caller may run both its up and its +/// down. A definition is run later by whoever applies or rolls back pending migrations, as the role +/// it declares, so gating only the run would let anyone plant SQL for a data table's admin to run. +async fn ensure_may_edit_migration( + db: &DB, + w_id: &str, + datatable_name: &str, + authed: &ApiAuthed, + migration: &str, + code_up: &str, + code_down: Option<&str>, +) -> Result<()> { + // Admin access reaches everything a role can, so it edits any definition without reading its + // annotation, which one saved before annotations were parsed may no longer pass. A refusal + // there would leave such a definition impossible to delete or fix. + let access = DatatableAccess::Authed(authed.to_authed_ref()); + if ensure_datatable_admin_access(db, w_id, datatable_name, &access) + .await + .is_ok() + { + return Ok(()); + } + for sql in std::iter::once(code_up).chain(code_down) { + ensure_migration_role_allowed(db, w_id, datatable_name, authed, sql, migration).await?; + } + Ok(()) +} + +/// Operators run deployed code only, and a migration is SQL written or run by hand. Checked before +/// anything else, including the role checks: on a data table not under roles those pass everyone. +fn ensure_not_operator(authed: &ApiAuthed) -> Result<()> { + if authed.is_operator { + return Err(Error::NotAuthorized( + "Operators cannot manage or run data table migrations".to_string(), + )); + } + Ok(()) +} + #[derive(Serialize)] struct AppliedMigration { version: i64, @@ -436,6 +473,7 @@ async fn run_datatable_migrations( Path((w_id, datatable_name)): Path<(String, String)>, Query(query): Query, ) -> JsonResult { + ensure_not_operator(&authed)?; // Before the admin connection is opened at all: the bookkeeping below is created and read // through it, so a caller no role covers must be refused here rather than after the fact. crate::datatable_permissions::ensure_reaches_datatable(&db, &w_id, &datatable_name, &authed) @@ -503,8 +541,7 @@ async fn run_datatable_migrations( &datatable_name, &authed, &m.code_up, - m.timestamp, - &m.name, + &format!("Migration {} ({})", m.timestamp, m.name), ) .await?; run_datatable_migration_job(&db, &user_db, &authed, &w_id, &database_arg, &m.code_up) @@ -573,6 +610,7 @@ async fn rollback_datatable_migrations( Path((w_id, datatable_name)): Path<(String, String)>, Query(query): Query, ) -> JsonResult { + ensure_not_operator(&authed)?; // Before the admin connection is opened at all: the bookkeeping below is created and read // through it, so a caller no role covers must be refused here rather than after the fact. crate::datatable_permissions::ensure_reaches_datatable(&db, &w_id, &datatable_name, &authed) @@ -666,8 +704,7 @@ async fn rollback_datatable_migrations( &datatable_name, &authed, &code_down, - version, - &definition.name, + &format!("Migration {} ({})", version, definition.name), ) .await?; @@ -1213,9 +1250,20 @@ async fn create_datatable_migration( Path((w_id, datatable_name)): Path<(String, String)>, Json(payload): Json, ) -> JsonResult { + ensure_not_operator(&authed)?; validate_datatable_path_segment(&datatable_name)?; validate_migration_name(&payload.name)?; ensure_datatable_migrations_enabled(&db, &w_id, &datatable_name).await?; + ensure_may_edit_migration( + &db, + &w_id, + &datatable_name, + &authed, + &format!("Migration '{}'", payload.name), + &payload.code_up, + payload.code_down.as_deref(), + ) + .await?; let mut tx = db.begin().await?; let timestamp = insert_datatable_migration_def( @@ -1267,6 +1315,34 @@ async fn delete_datatable_migration( Extension(db): Extension, Path((w_id, datatable_name, timestamp)): Path<(String, String, i64)>, ) -> Result { + ensure_not_operator(&authed)?; + let deleted_message = format!("Deleted migration {} from {}", timestamp, datatable_name); + + // A definition already gone is not an error: `wmill sync push` deletes whatever the repo + // dropped, which may no longer exist here. + let Some(definition) = sqlx::query!( + "SELECT name, code_up, code_down FROM datatable_migrations \ + WHERE workspace_id = $1 AND datatable = $2 AND timestamp = $3", + &w_id, + &datatable_name, + timestamp, + ) + .fetch_optional(&db) + .await? + else { + return Ok(deleted_message); + }; + ensure_may_edit_migration( + &db, + &w_id, + &datatable_name, + &authed, + &format!("Migration {} ({})", timestamp, definition.name), + &definition.code_up, + definition.code_down.as_deref(), + ) + .await?; + // Hold the run-serialization lock across the applied-check and the delete: a // run snapshots a migration's SQL before recording its version, so an // unserialized delete could race it and leave `_wm_migrations` pointing at a @@ -1294,19 +1370,46 @@ async fn delete_datatable_migration( ))); } + // Delete only the SQL the role check above cleared: it read the definition without the lock, + // and a rewrite that landed since was never checked. let deleted_name = sqlx::query_scalar!( "DELETE FROM datatable_migrations \ WHERE workspace_id = $1 AND datatable = $2 AND timestamp = $3 \ + AND code_up = $4 AND code_down IS NOT DISTINCT FROM $5 \ RETURNING name", &w_id, &datatable_name, timestamp, + &definition.code_up, + definition.code_down.as_deref(), ) .fetch_optional(&db) .await?; // The definition is removed; runs may resume (audit/deploy metadata below // don't need the lock). drop(lock_client); + let Some(name) = deleted_name else { + // Nothing matched either because another delete got there first, which is this one's + // outcome too, or because the definition was rewritten. + let still_defined = sqlx::query_scalar!( + "SELECT EXISTS(SELECT 1 FROM datatable_migrations \ + WHERE workspace_id = $1 AND datatable = $2 AND timestamp = $3)", + &w_id, + &datatable_name, + timestamp, + ) + .fetch_one(&db) + .await? + .unwrap_or(false); + if !still_defined { + return Ok(deleted_message); + } + return Err(Error::BadRequest(format!( + "Migration {} on data table '{}' changed while it was being deleted. Reload it and \ + retry.", + timestamp, datatable_name + ))); + }; audit_log( &db, @@ -1319,23 +1422,10 @@ async fn delete_datatable_migration( ) .await?; - // Only tally a change if a migration was actually deleted. - if let Some(name) = deleted_name { - record_datatable_migration_deployment( - &authed, - &db, - &w_id, - &datatable_name, - timestamp, - &name, - ) + record_datatable_migration_deployment(&authed, &db, &w_id, &datatable_name, timestamp, &name) .await?; - } - Ok(format!( - "Deleted migration {} from {}", - timestamp, datatable_name - )) + Ok(deleted_message) } #[derive(Deserialize)] @@ -1356,6 +1446,7 @@ async fn upsert_datatable_migration( Path((w_id, datatable_name)): Path<(String, String)>, Json(payload): Json, ) -> Result { + ensure_not_operator(&authed)?; validate_datatable_path_segment(&datatable_name)?; validate_migration_name(&payload.name)?; ensure_datatable_migrations_enabled(&db, &w_id, &datatable_name).await?; @@ -1381,9 +1472,9 @@ async fn upsert_datatable_migration( // run-serialization lock across the applied-check and the write below so an // in-flight run can't record a version for the SQL we're about to overwrite. // Held until the end of the handler (well past the write). The exempt - // upserts need no lock: a new or unchanged one overwrites nothing, and one - // that only adds a down leaves the `code_up` a concurrent run is recording - // a version for untouched. + // upserts need no lock: a new one overwrites nothing, and one that only adds + // a down leaves the `code_up` a concurrent run is recording a version for + // untouched. let only_adds_down = existing.as_ref().is_some_and(|existing| { existing.name == payload.name && existing.code_up == payload.code_up @@ -1395,8 +1486,40 @@ async fn upsert_datatable_migration( && existing.code_up == payload.code_up && existing.code_down == payload.code_down }); + let upserted_message = format!( + "Upserted migration {} in {}", + payload.timestamp, datatable_name + ); + // A re-push of the definition as stored changes nothing, so it is not checked, written, audited, + // synced or counted. Writing it anyway would re-create a definition deleted since the read above. + if unchanged { + return Ok(upserted_message); + } + ensure_may_edit_migration( + &db, + &w_id, + &datatable_name, + &authed, + &format!("Migration {} ({})", payload.timestamp, payload.name), + &payload.code_up, + payload.code_down.as_deref(), + ) + .await?; + // Replacing a definition removes the one stored, so it answers to the same check as a delete. + if let Some(existing) = existing.as_ref() { + ensure_may_edit_migration( + &db, + &w_id, + &datatable_name, + &authed, + &format!("Migration {} ({})", payload.timestamp, existing.name), + &existing.code_up, + existing.code_down.as_deref(), + ) + .await?; + } let _run_lock = match existing.as_ref() { - Some(_) if !only_adds_down && !unchanged => { + Some(_) if !only_adds_down => { // Fail closed: if we can't lock/read the applied set (e.g. the // data-table database is temporarily unreachable), refuse the change // rather than risk overwriting a migration that has already run. @@ -1426,31 +1549,27 @@ async fn upsert_datatable_migration( _ => None, }; - // An exempt upsert judged the row from an unlocked read and then writes - // without the lock, so that whole read is re-tested here, where `ON CONFLICT - // DO UPDATE` re-reads the row under a row lock. Otherwise a request working - // from a stale definition silently reverts whatever changed in between — a - // second addition's down, or a locked rewrite of the up whose new SQL a run - // may already have recorded a version for. Reading no row at all is part of - // the premise: the equality is NULL when `$8` is, so a version created in the - // meantime is refused rather than overwritten. - let recheck_observed = _run_lock.is_none(); + // The role checks, and the exemptions from the lock, all judged the row from the unlocked read + // above, so that whole read is re-tested here, where `ON CONFLICT DO UPDATE` re-reads the row + // under a row lock. Otherwise a request working from a stale definition silently replaces + // whatever changed in between: SQL the role checks never saw, or a locked rewrite of the up + // whose new SQL a run may already have recorded a version for. Reading no row at all is part of + // the premise: the equality is NULL when `$7` is, so a version created in the meantime is + // refused rather than overwritten. let written = sqlx::query!( "INSERT INTO datatable_migrations (workspace_id, datatable, timestamp, name, code_up, code_down) \ VALUES ($1, $2, $3, $4, $5, $6) \ ON CONFLICT (workspace_id, datatable, timestamp) DO UPDATE \ SET name = EXCLUDED.name, code_up = EXCLUDED.code_up, code_down = EXCLUDED.code_down \ - WHERE NOT $7 \ - OR (datatable_migrations.name = $8::text \ - AND datatable_migrations.code_up = $9::text \ - AND datatable_migrations.code_down IS NOT DISTINCT FROM $10::text)", + WHERE datatable_migrations.name = $7::text \ + AND datatable_migrations.code_up = $8::text \ + AND datatable_migrations.code_down IS NOT DISTINCT FROM $9::text", &w_id, &datatable_name, payload.timestamp, &payload.name, &payload.code_up, payload.code_down.as_deref(), - recheck_observed, existing.as_ref().map(|e| e.name.as_str()), existing.as_ref().map(|e| e.code_up.as_str()), existing.as_ref().and_then(|e| e.code_down.as_deref()), @@ -1489,24 +1608,17 @@ async fn upsert_datatable_migration( ) .await?; - // An unchanged re-push is not counted: `wmill sync push` sends every migration - // on every sync, so counting those would swamp the definitions people write. - if !unchanged { - windmill_common::feature_usage::log_feature_usage( - "datatable", - "migration_created", - if existing.is_none() { - "synced" - } else { - "edited" - }, - ); - } + windmill_common::feature_usage::log_feature_usage( + "datatable", + "migration_created", + if existing.is_none() { + "synced" + } else { + "edited" + }, + ); - Ok(format!( - "Upserted migration {} in {}", - payload.timestamp, datatable_name - )) + Ok(upserted_message) } /// Generate the first migration for a data table by snapshotting its current @@ -1519,6 +1631,7 @@ async fn generate_initial_datatable_migration( Extension(db): Extension, Path((w_id, datatable_name)): Path<(String, String)>, ) -> JsonResult { + ensure_not_operator(&authed)?; // Returns a `pg_dump` of the whole schema and writes into the data table's own bookkeeping, so // it answers to the workspace that governs it rather than to whoever is asking. ensure_datatable_admin_access( diff --git a/backend/windmill-api-workspaces/src/datatable_permissions.rs b/backend/windmill-api-workspaces/src/datatable_permissions.rs index 5cb1f3c1c3..69e04e328d 100644 --- a/backend/windmill-api-workspaces/src/datatable_permissions.rs +++ b/backend/windmill-api-workspaces/src/datatable_permissions.rs @@ -65,3 +65,15 @@ pub(crate) async fn ensure_reaches_datatable( ) -> Result<()> { roles::ensure_reaches_datatable(db, w_id, datatable_name, authed).await } + +/// [`ensure_reaches_datatable`] against an entry already resolved, for a caller that goes on to +/// connect from that same entry. +pub(crate) async fn ensure_reaches_governing_datatable( + db: &DB, + w_id: &str, + datatable_name: &str, + governing: &GoverningDatatable, + authed: &ApiAuthed, +) -> Result<()> { + roles::ensure_reaches_governing_datatable(db, w_id, datatable_name, governing, authed).await +} diff --git a/backend/windmill-api-workspaces/src/datatable_permissions_oss.rs b/backend/windmill-api-workspaces/src/datatable_permissions_oss.rs index f4d8c6a7ad..736493cc2d 100644 --- a/backend/windmill-api-workspaces/src/datatable_permissions_oss.rs +++ b/backend/windmill-api-workspaces/src/datatable_permissions_oss.rs @@ -12,8 +12,9 @@ #[cfg(all(feature = "private", feature = "enterprise"))] pub(crate) use crate::datatable_permissions_ee::{ - ensure_governs_datatable, ensure_reaches_datatable, get_datatable_permissions, - list_usable_datatable_roles, set_datatable_permissions, + ensure_governs_datatable, ensure_reaches_datatable, ensure_reaches_governing_datatable, + get_datatable_permissions, list_usable_datatable_roles, set_datatable_permissions, + usable_datatable_roles, }; #[cfg(not(all(feature = "private", feature = "enterprise")))] @@ -56,6 +57,20 @@ mod ce { } } + pub(crate) async fn ensure_reaches_governing_datatable( + _db: &DB, + _w_id: &str, + _datatable_name: &str, + governing: &GoverningDatatable, + _authed: &ApiAuthed, + ) -> Result<()> { + if governing.datatable.permissions.is_none() { + Ok(()) + } else { + Err(unavailable()) + } + } + // The routes stay registered so the API has one shape; each answers after authentication, // before anything is read. @@ -70,4 +85,28 @@ mod ce { pub(crate) async fn list_usable_datatable_roles(_authed: ApiAuthed) -> Result { Err(unavailable()) } + + pub(crate) struct UsableDatatableRoles { + pub(crate) permissioned: bool, + pub(crate) roles: Vec, + pub(crate) default_role: String, + } + + /// A data table not under roles is used as `admin`, as before roles existed. One under roles + /// is refused: no role of it can be connected as. + pub(crate) async fn usable_datatable_roles( + _db: &DB, + _authed: &ApiAuthed, + _w_id: &str, + governing: &GoverningDatatable, + ) -> Result { + if governing.datatable.permissions.is_some() { + return Err(unavailable()); + } + Ok(UsableDatatableRoles { + permissioned: false, + roles: vec![], + default_role: windmill_common::datatable_roles::ADMIN_DATATABLE_ROLE.to_string(), + }) + } } diff --git a/backend/windmill-api-workspaces/src/datatable_replay_oss.rs b/backend/windmill-api-workspaces/src/datatable_replay_oss.rs new file mode 100644 index 0000000000..edcf8e402b --- /dev/null +++ b/backend/windmill-api-workspaces/src/datatable_replay_oss.rs @@ -0,0 +1,48 @@ +/* + * Author: Ruben Fiszel + * Copyright: Windmill Labs, Inc 2022 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + +//! Where the replay of a data table's owners and grants into its copy comes from: the enterprise +//! one, or a refusal. Without it a data table under roles is not copied at all: its rows would +//! arrive owned by the admin connection with no grant for any role. + +#[cfg(all(feature = "private", feature = "enterprise"))] +pub(crate) use crate::datatable_replay_ee::replay_owners_and_grants; + +#[cfg(all(feature = "private", feature = "enterprise"))] +pub(crate) fn ensure_replay() -> windmill_common::error::Result<()> { + Ok(()) +} + +#[cfg(not(all(feature = "private", feature = "enterprise")))] +use { + std::collections::BTreeSet, + tokio::sync::mpsc, + tokio_postgres::error::DbError, + windmill_common::error::{Error, Result}, +}; + +/// Checked before anything is created. +#[cfg(not(all(feature = "private", feature = "enterprise")))] +pub(crate) fn ensure_replay() -> Result<()> { + Err(Error::BadRequest( + "Cloning a data table under roles is a Windmill Enterprise Edition feature: the copy \ + needs the source's owners and grants replayed. Fork it keeping the original database \ + instead." + .to_string(), + )) +} + +#[cfg(not(all(feature = "private", feature = "enterprise")))] +pub(crate) async fn replay_owners_and_grants( + _source: &tokio_postgres::Client, + _target: &tokio_postgres::Transaction<'_>, + _notices: &mut mpsc::UnboundedReceiver, + _catalog_roles: &BTreeSet, +) -> Result<()> { + ensure_replay() +} diff --git a/backend/windmill-api-workspaces/src/lib.rs b/backend/windmill-api-workspaces/src/lib.rs index c1eb6c1ffc..c437d7e2b4 100644 --- a/backend/windmill-api-workspaces/src/lib.rs +++ b/backend/windmill-api-workspaces/src/lib.rs @@ -1,10 +1,15 @@ #[cfg(feature = "parquet")] pub mod ai_session_backups; pub mod data_metrics; +pub mod datatable_acl; +pub mod datatable_acl_oss; +pub mod datatable_clone; pub mod datatable_migrations; pub mod datatable_permissions; pub mod datatable_permissions_oss; +pub mod datatable_replay_oss; pub mod deployment_requests; +pub mod remote_deploy; pub mod workspaces; pub mod workspaces_extra; pub mod workspaces_oss; @@ -12,5 +17,10 @@ pub mod workspaces_oss; #[cfg(feature = "private")] pub mod workspaces_ee; +#[cfg(all(feature = "private", feature = "enterprise"))] +pub mod datatable_acl_ee; +#[cfg(all(feature = "private", feature = "enterprise"))] +pub mod datatable_replay_ee; + #[cfg(all(feature = "private", feature = "enterprise"))] pub mod datatable_permissions_ee; diff --git a/backend/windmill-api-workspaces/src/remote_deploy.rs b/backend/windmill-api-workspaces/src/remote_deploy.rs new file mode 100644 index 0000000000..3e701cbf14 --- /dev/null +++ b/backend/windmill-api-workspaces/src/remote_deploy.rs @@ -0,0 +1,794 @@ +/* + * Author: Ruben Fiszel + * Copyright: Windmill Labs, Inc 2026 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + +//! Deploying from the UI into a workspace of another Windmill instance. +//! +//! The browser runs the same deploy it runs between two workspaces of this instance, and sends +//! every call aimed at the target through [`proxy`], which forwards it to the remote workspace +//! with the caller's own token for that instance. The remote authorizes and audits the deploy as +//! that person, so nothing here needs to know what a deploy is made of. + +use std::time::Duration; + +use axum::{ + body::{Body, Bytes}, + extract::{DefaultBodyLimit, Extension, OriginalUri, Path}, + http::{header, HeaderMap, Method, StatusCode}, + response::Response, + routing::{any, get, post}, + Json, Router, +}; +use chrono::{DateTime, Utc}; +use lazy_static::lazy_static; +use magic_crypt::MagicCrypt256; +use regex::Regex; +use serde::{Deserialize, Serialize}; +use windmill_api_auth::{is_effectively_unscoped, ApiAuthed, Tokened}; +use windmill_audit::{audit_oss::audit_log, ActionKind}; +use windmill_common::{ + auth::hash_token, + error::{error_source_chain, Error, Result}, + ssrf::validate_url_for_ssrf, + utils::{configure_client, rd_string, require_admin}, + variables::{build_crypt, decrypt, encrypt}, + worker::CLOUD_HOSTED, + DB, +}; + +lazy_static! { + static ref REMOTE_WORKSPACE_ID: Regex = Regex::new("^[a-zA-Z0-9_-]{1,50}$").unwrap(); + /// Shared so that a deploy's calls reuse connections. A cloud instance instead builds a client + /// per request, pinned to the addresses it just validated for that target. + static ref REMOTE_CLIENT: reqwest::Client = remote_client_builder().build().unwrap(); +} + +const PROXY_PREFIX: &str = "/remote_deploy/proxy/"; + +pub fn workspaced_service(proxy_body_limit: usize) -> Router { + Router::new() + .route("/target", get(get_target).post(set_target)) + .route("/connect", post(connect)) + .route("/disconnect", post(disconnect)) + // `{key}` is the caller's `proxy_key`. The session cookie is `SameSite=Lax`, so it rides a + // top-level navigation from any site: without a value such a link cannot know, it could + // spend the stored token, to run something on the remote or to render its content on this + // origin. `Sec-Fetch-Site` is no substitute, since plain-http instances never receive it. + .route( + "/proxy/{key}/{*rest}", + any(proxy).layer(DefaultBodyLimit::max(proxy_body_limit)), + ) +} + +/// A request URI as logs should record it: with the proxy key masked, since that key is what keeps +/// a link from spending a stored remote token. +pub struct RedactedUri<'a>(pub &'a axum::http::Uri); + +impl std::fmt::Display for RedactedUri<'_> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let Some((head, keyed)) = self.0.path().split_once(PROXY_PREFIX) else { + return self.0.fmt(f); + }; + let rest = keyed.find('/').map_or("", |i| &keyed[i..]); + write!(f, "{head}{PROXY_PREFIX}***{rest}")?; + match self.0.query() { + Some(query) => write!(f, "?{query}"), + None => Ok(()), + } + } +} + +#[derive(Serialize, Deserialize, Clone, Debug)] +pub struct RemoteDeployTarget { + /// Root URL of the remote instance, without `/api`. + pub base_url: String, + pub workspace_id: String, +} + +impl RemoteDeployTarget { + fn normalized(self) -> Result { + let base_url = self.base_url.trim().trim_end_matches('/').to_string(); + let parsed = url::Url::parse(&base_url) + .map_err(|e| Error::BadRequest(format!("Invalid remote instance URL: {e}")))?; + if !matches!(parsed.scheme(), "http" | "https") || parsed.host_str().is_none() { + return Err(Error::BadRequest( + "The remote instance URL must be an http(s) URL with a host".to_string(), + )); + } + if !parsed.username().is_empty() + || parsed.password().is_some() + || parsed.query().is_some() + || parsed.fragment().is_some() + { + return Err(Error::BadRequest( + "The remote instance URL must not carry credentials, a query or a fragment" + .to_string(), + )); + } + let workspace_id = self.workspace_id.trim().to_string(); + if !REMOTE_WORKSPACE_ID.is_match(&workspace_id) { + return Err(Error::BadRequest(format!( + "Invalid remote workspace id: {workspace_id}" + ))); + } + Ok(Self { base_url, workspace_id }) + } + + fn api_url(&self, suffix: &str) -> String { + format!("{}/api/w/{}/{suffix}", self.base_url, self.workspace_id) + } +} + +#[derive(Serialize)] +struct RemoteDeployConnection { + remote_email: String, + /// Goes in every proxy URL, see [`workspaced_service`]. + proxy_key: String, + connected_at: DateTime, +} + +#[derive(Serialize)] +struct RemoteDeployStatus { + target: Option, + /// The caller's own connection to `target`. + connection: Option, +} + +/// A stored token acts on another instance as its owner, with none of the restrictions of the +/// credential that reaches it. Only the owner acting directly with full rights may use or replace +/// it: a job token runs as whoever a `wm_deployers` member pointed `on_behalf_of` at, and a +/// scoped or read-only token was never granted this. +fn require_own_credentials(authed: &ApiAuthed) -> Result<()> { + if authed.job_id.is_some() + || authed.read_only + || !is_effectively_unscoped(authed.scopes.as_deref()) + { + return Err(Error::PermissionDenied( + "Deploying to a remote instance requires your own session or an unscoped token, \ + not a job, scoped or read-only token" + .to_string(), + )); + } + Ok(()) +} + +fn parse_target(target: Option) -> Result> { + target + .map(serde_json::from_value) + .transpose() + .map_err(|e| Error::internal_err(format!("reading the remote deploy target: {e}"))) +} + +fn no_target(w_id: &str) -> Error { + Error::BadRequest(format!( + "Workspace {w_id} has no remote deploy target. An admin sets it in the workspace \ + settings, under Dev workspace" + )) +} + +async fn load_target(db: &DB, w_id: &str) -> Result> { + let target = sqlx::query_scalar!( + "SELECT remote_deploy_target FROM workspace_settings WHERE workspace_id = $1", + w_id + ) + .fetch_optional(db) + .await? + .flatten(); + parse_target(target) +} + +async fn require_target(db: &DB, w_id: &str) -> Result { + load_target(db, w_id).await?.ok_or_else(|| no_target(w_id)) +} + +fn remote_client_builder() -> reqwest::ClientBuilder { + configure_client(reqwest::ClientBuilder::new()) + .user_agent("windmill/remote-deploy") + .redirect(reqwest::redirect::Policy::none()) + .connect_timeout(Duration::from_secs(10)) + .timeout(Duration::from_secs(120)) +} + +async fn remote_client(target: &RemoteDeployTarget) -> Result { + // A workspace admin chose the URL. A self-hosted instance may deploy into its own private + // network, as it may reach a private git remote; a cloud workspace must not. + if !*CLOUD_HOSTED { + return Ok(REMOTE_CLIENT.clone()); + } + validate_url_for_ssrf(&target.base_url) + .await? + .apply_dns_pinning(remote_client_builder()) + .build() + .map_err(|e| Error::internal_err(format!("building the remote deploy client: {e}"))) +} + +fn unreachable(target: &RemoteDeployTarget, e: reqwest::Error) -> Error { + Error::BadGateway(format!( + "Could not reach the remote instance {}: {}", + target.base_url, + error_source_chain(&e) + )) +} + +/// Responses carrying a `proxy_key` must not be kept by any cache between here and the browser. +const NO_STORE: [(header::HeaderName, &str); 1] = [(header::CACHE_CONTROL, "no-store")]; + +struct StoredConnection { + token: String, + remote_email: String, + proxy_key: String, + connected_at: DateTime, +} + +/// The caller's connection to `target`, if they may still use it. +/// +/// `connect` takes no lock against what clears these rows (a removal from the workspace, a target +/// change, a key rotation): writers take those rows in every order, so any lock it held could close +/// a deadlock. A row can therefore land after one of them ran, however long its remote call took, +/// and is voided here instead, by identity rather than by comparing times: it counts only while +/// the target setting and the owner's membership are the very ones it was connected under (see +/// `member_since`, `target_changed_at`) or the owner is a superadmin, and while it decrypts. An +/// emptied row is a disconnect's. +async fn load_connection( + db: &DB, + w_id: &str, + email: &str, + target: &RemoteDeployTarget, +) -> Result> { + let Some(row) = sqlx::query_as!( + StoredConnection, + "SELECT t.token, t.remote_email, t.proxy_key, t.connected_at FROM remote_deploy_token t + JOIN workspace_settings s ON s.workspace_id = t.workspace_id + WHERE t.workspace_id = $1 AND t.email = $2 AND t.base_url = $3 + AND t.remote_workspace_id = $4 AND t.token <> '' + AND s.remote_deploy_target_changed_at IS NOT DISTINCT FROM t.target_changed_at + AND (EXISTS (SELECT 1 FROM usr u WHERE u.workspace_id = t.workspace_id + AND u.email = t.email AND u.created_at = t.member_since) + OR EXISTS (SELECT 1 FROM password p WHERE p.email = t.email AND p.super_admin))", + w_id, + email, + &target.base_url, + &target.workspace_id + ) + .fetch_optional(db) + .await? + else { + return Ok(None); + }; + match decrypt(&build_crypt(db, w_id).await?, row.token) { + Ok(token) => Ok(Some(StoredConnection { token, ..row })), + Err(e) => { + tracing::warn!("remote deploy token of {email} in {w_id} does not decrypt: {e}"); + Ok(None) + } + } +} + +async fn get_target( + authed: ApiAuthed, + Extension(db): Extension, + Path(w_id): Path, +) -> Result<( + [(header::HeaderName, &'static str); 1], + Json, +)> { + let target = load_target(&db, &w_id).await?; + // The connection carries the proxy key. A credential that may not use the proxy (see + // `require_own_credentials`) must not read it either: it could hand its owner a working link. + let connection = match &target { + Some(target) if require_own_credentials(&authed).is_ok() => { + load_connection(&db, &w_id, &authed.email, target) + .await? + .map(|c| RemoteDeployConnection { + remote_email: c.remote_email, + proxy_key: c.proxy_key, + connected_at: c.connected_at, + }) + } + _ => None, + }; + Ok((NO_STORE, Json(RemoteDeployStatus { target, connection }))) +} + +#[derive(Deserialize)] +struct SetRemoteDeployTarget { + target: Option, +} + +async fn set_target( + authed: ApiAuthed, + Extension(db): Extension, + Path(w_id): Path, + Json(request): Json, +) -> Result { + if !cfg!(feature = "enterprise") { + return Err(Error::BadRequest( + "Deploying to another instance is only available on Windmill Enterprise Edition" + .to_string(), + )); + } + // Everyone who connects afterwards hands their token to this URL. + require_own_credentials(&authed)?; + require_admin(authed.is_admin, &authed.username)?; + let target = request + .target + .map(RemoteDeployTarget::normalized) + .transpose()?; + + let mut tx = db.begin().await?; + let (base_url, remote_workspace_id) = match &target { + Some(t) => (Some(t.base_url.as_str()), Some(t.workspace_id.as_str())), + None => (None, None), + }; + audit_log( + &mut *tx, + &authed, + "workspaces.edit_remote_deploy_target", + ActionKind::Update, + &w_id, + None, + Some( + [ + ("base_url", base_url.unwrap_or("")), + ("remote_workspace_id", remote_workspace_id.unwrap_or("")), + ] + .into(), + ), + ) + .await?; + let target_json = target + .as_ref() + .map(serde_json::to_value) + .transpose() + .map_err(|e| Error::internal_err(e.to_string()))?; + sqlx::query!( + "UPDATE workspace_settings SET remote_deploy_target = $1::jsonb, + remote_deploy_target_changed_at = CASE + WHEN remote_deploy_target IS DISTINCT FROM $1::jsonb THEN clock_timestamp() + ELSE remote_deploy_target_changed_at END + WHERE workspace_id = $2", + target_json, + &w_id + ) + .execute(&mut *tx) + .await?; + // A token is only ever sent to the target it was granted for, so the ones for any other + // target are dead credentials. + sqlx::query!( + "DELETE FROM remote_deploy_token WHERE workspace_id = $1 + AND ($2::text IS NULL OR base_url <> $2 OR remote_workspace_id <> $3)", + &w_id, + base_url, + remote_workspace_id + ) + .execute(&mut *tx) + .await?; + tx.commit().await?; + + Ok(match target { + Some(t) => format!( + "Workspace {w_id} deploys to {} on {}", + t.workspace_id, t.base_url + ), + None => format!("Removed the remote deploy target of workspace {w_id}"), + }) +} + +#[derive(Deserialize)] +struct ConnectRequest { + token: String, + /// The target the caller got the token for, as the drawer showed it. + target: RemoteDeployTarget, +} + +#[derive(Deserialize)] +struct RemoteWhoami { + email: String, +} + +async fn connect( + authed: ApiAuthed, + Tokened { token: credential }: Tokened, + Extension(db): Extension, + Path(w_id): Path, + Json(request): Json, +) -> Result<( + [(header::HeaderName, &'static str); 1], + Json, +)> { + require_own_credentials(&authed)?; + // Read before the remote call, which can take long enough for any of it to change: the row is + // bound to the target version and the membership seen here (see `load_connection`), and + // stamped with when this connect started, so that a disconnect or a newer connect made while + // it runs keeps it from landing. + let start = sqlx::query!( + r#"SELECT clock_timestamp() AS "started_at!", s.remote_deploy_target, + s.remote_deploy_target_changed_at, + (SELECT u.created_at FROM usr u + WHERE u.workspace_id = s.workspace_id AND u.email = $2) AS member_since + FROM workspace_settings s WHERE s.workspace_id = $1"#, + &w_id, + &authed.email + ) + .fetch_optional(&db) + .await? + .ok_or_else(|| no_target(&w_id))?; + let target = parse_target(start.remote_deploy_target)?.ok_or_else(|| no_target(&w_id))?; + // A token is only ever sent to the instance it was meant for. Without this, re-pointing the + // target between the user getting a token and posting it here would hand it to the new one; + // after this check the token still goes to `target` only, never to what the setting became. + let requested = request.target.normalized()?; + if requested.base_url != target.base_url || requested.workspace_id != target.workspace_id { + return Err(Error::BadRequest( + "The remote deploy target changed since you started connecting; reload and connect \ + again" + .to_string(), + )); + } + let token = request.token.trim(); + if token.is_empty() { + return Err(Error::BadRequest("The token is empty".to_string())); + } + + let response = remote_client(&target) + .await? + .get(target.api_url("users/whoami")) + .bearer_auth(token) + .send() + .await + .map_err(|e| unreachable(&target, e))?; + let status = response.status(); + if !status.is_success() { + let body = response.text().await.unwrap_or_default(); + return Err(Error::BadRequest(format!( + "{} did not accept this token for workspace {} ({status}): {body}", + target.base_url, target.workspace_id + ))); + } + let whoami: RemoteWhoami = response.json().await.map_err(|e| { + Error::BadGateway(format!( + "{} did not answer like a Windmill instance: {}", + target.base_url, + error_source_chain(&e) + )) + })?; + + let proxy_key = rd_string(32); + let mc = build_crypt(&db, &w_id).await?; + // No lock against removals, target changes or key rotations: `load_connection` voids a row + // that lands after one of them. A disconnect or a newer connect stamps the row itself, which + // this one then leaves alone. Without a membership to bind to, the row is bound to the account + // through the credential making this request, which deleting the account removes (except + // through `leave_instance`, which leaves its tokens): the address could otherwise be deleted + // and taken by a new account while the remote call runs, and the foreign key would accept it. + let mut tx = db.begin().await?; + let connected_at = sqlx::query_scalar!( + "INSERT INTO remote_deploy_token + (workspace_id, email, base_url, remote_workspace_id, token, remote_email, proxy_key, + connected_at, member_since, target_changed_at) + SELECT $1, $2, $3, $4, $5, $6, $7, $8, $9, $10 + WHERE $9::timestamptz IS NOT NULL + OR EXISTS (SELECT 1 FROM token WHERE token_hash = $11) + ON CONFLICT (workspace_id, email) DO UPDATE SET + base_url = EXCLUDED.base_url, remote_workspace_id = EXCLUDED.remote_workspace_id, + token = EXCLUDED.token, remote_email = EXCLUDED.remote_email, + proxy_key = EXCLUDED.proxy_key, connected_at = EXCLUDED.connected_at, + member_since = EXCLUDED.member_since, target_changed_at = EXCLUDED.target_changed_at + WHERE remote_deploy_token.connected_at < EXCLUDED.connected_at + RETURNING connected_at", + &w_id, + &authed.email, + &target.base_url, + &target.workspace_id, + encrypt(&mc, token), + &whoami.email, + &proxy_key, + start.started_at, + start.member_since, + start.remote_deploy_target_changed_at, + hash_token(&credential) + ) + .fetch_optional(&mut *tx) + .await? + .ok_or_else(|| { + Error::BadRequest(match start.member_since { + Some(_) => "This connect was not saved: you disconnected or connected again while it \ + was running" + .to_string(), + None => format!( + "This connect was not saved: you disconnected, connected again or signed out \ + while it was running. Connecting workspace {w_id} without being one of its \ + members needs a session or an API token of this instance" + ), + }) + })?; + audit_log( + &mut *tx, + &authed, + "workspaces.remote_deploy_connect", + ActionKind::Create, + &w_id, + Some(&whoami.email), + Some([("base_url", target.base_url.as_str())].into()), + ) + .await?; + tx.commit().await?; + + Ok(( + NO_STORE, + Json(RemoteDeployConnection { remote_email: whoami.email, proxy_key, connected_at }), + )) +} + +async fn disconnect( + authed: ApiAuthed, + Extension(db): Extension, + Path(w_id): Path, +) -> Result { + require_own_credentials(&authed)?; + let mut tx = db.begin().await?; + // Emptied rather than deleted, and stamped, even with no connection yet: a connect that started + // before this must not bring one back when it lands (see `connect`). + sqlx::query!( + "INSERT INTO remote_deploy_token + (workspace_id, email, base_url, remote_workspace_id, token, remote_email, proxy_key, + connected_at) + VALUES ($1, $2, '', '', '', '', '', clock_timestamp()) + ON CONFLICT (workspace_id, email) DO UPDATE SET + token = '', proxy_key = '', connected_at = clock_timestamp()", + &w_id, + &authed.email + ) + .execute(&mut *tx) + .await?; + audit_log( + &mut *tx, + &authed, + "workspaces.remote_deploy_disconnect", + ActionKind::Delete, + &w_id, + None, + None, + ) + .await?; + tx.commit().await?; + Ok(format!( + "Disconnected from the remote deploy target of {w_id}" + )) +} + +/// The remote URL for the request path after the proxy prefix and key, as the client sent it. +/// +/// `url` rewrites a path while parsing it: it resolves dot segments (`%2e` spellings included) +/// and turns backslashes into slashes. Either would reach a route other than the one this +/// instance authorized — outside the remote workspace, or a route its read-only check does not +/// recognize — so a path the parser would change is refused rather than forwarded. +fn forwarded_url( + target: &RemoteDeployTarget, + original_path: &str, + query: Option<&str>, +) -> Result { + let suffix = original_path + .split_once(PROXY_PREFIX) + .and_then(|(_, keyed)| keyed.split_once('/')) + .map(|(_, suffix)| suffix) + .unwrap_or_default(); + let invalid = || Error::BadRequest(format!("Invalid remote deploy path: {suffix}")); + let base_path = url::Url::parse(&target.base_url) + .map_err(|_| invalid())? + .path() + .trim_end_matches('/') + .to_string(); + let mut url = url::Url::parse(&target.api_url(suffix)).map_err(|_| invalid())?; + let expected_path = format!("{base_path}/api/w/{}/{suffix}", target.workspace_id); + if suffix.is_empty() || url.path() != expected_path { + return Err(invalid()); + } + url.set_query(query); + Ok(url) +} + +async fn proxy( + authed: ApiAuthed, + Extension(db): Extension, + Path((w_id, key, _rest)): Path<(String, String, String)>, + OriginalUri(uri): OriginalUri, + method: Method, + headers: HeaderMap, + body: Bytes, +) -> Result { + require_own_credentials(&authed)?; + let target = require_target(&db, &w_id).await?; + let url = forwarded_url(&target, uri.path(), uri.query())?; + let stored = load_connection(&db, &w_id, &authed.email, &target) + .await? + .ok_or_else(|| { + Error::BadRequest(format!( + "Connect to {} with your own token before deploying there", + target.base_url + )) + })?; + if !constant_time_eq::constant_time_eq(key.as_bytes(), stored.proxy_key.as_bytes()) { + return Err(Error::BadRequest( + "This remote deploy link is not yours or is out of date; reload the page".to_string(), + )); + } + + // Only what describes the payload: the caller's cookie and token belong to this instance. + let mut request = remote_client(&target) + .await? + .request(method, url) + .bearer_auth(stored.token) + .body(body); + for name in [header::CONTENT_TYPE, header::ACCEPT] { + if let Some(value) = headers.get(&name) { + request = request.header(name, value.clone()); + } + } + let response = request.send().await.map_err(|e| unreachable(&target, e))?; + + let status = response.status(); + // Passed through, a 401 would read as this instance's session having expired and log the + // user out of it. + if status == StatusCode::UNAUTHORIZED { + let body = response.text().await.unwrap_or_default(); + return Err(Error::BadGateway(format!( + "{} rejected your stored token: {body}", + target.base_url + ))); + } + if status.is_redirection() { + let location = response + .headers() + .get(header::LOCATION) + .and_then(|l| l.to_str().ok()) + .unwrap_or_default() + .to_string(); + return Err(Error::BadGateway(format!( + "{} redirected to {location}. Set the remote deploy target to the URL it redirects to", + target.base_url + ))); + } + // The body is the remote's and may be anything its users stored (an uploaded file, a job + // result typed `text/html`); served from this origin it must never render as a page here. + let mut builder = Response::builder() + .status(status) + .header(header::X_CONTENT_TYPE_OPTIONS, "nosniff") + .header( + header::CONTENT_SECURITY_POLICY, + "sandbox; default-src 'none'", + ); + if let Some(content_type) = response.headers().get(header::CONTENT_TYPE) { + builder = builder.header(header::CONTENT_TYPE, content_type); + } + builder + .body(Body::from_stream(response.bytes_stream())) + .map_err(|e| Error::internal_err(format!("building the proxied response: {e}"))) +} + +/// Move the stored remote tokens to a new workspace key. A token that no longer decrypts is +/// dropped: its owner connects again. A disconnect's emptied row is kept, as the stamp that keeps +/// an older connect from landing. +pub(crate) async fn reencrypt_tokens( + conn: &mut sqlx::PgConnection, + w_id: &str, + old: &MagicCrypt256, + new: &MagicCrypt256, +) -> Result<()> { + let rows = sqlx::query!( + "SELECT email, token FROM remote_deploy_token + WHERE workspace_id = $1 AND token <> '' FOR UPDATE", + w_id + ) + .fetch_all(&mut *conn) + .await?; + for row in rows { + match decrypt(old, row.token) { + Ok(plain) => { + sqlx::query!( + "UPDATE remote_deploy_token SET token = $1 + WHERE workspace_id = $2 AND email = $3", + encrypt(new, &plain), + w_id, + row.email + ) + .execute(&mut *conn) + .await?; + } + Err(_) => { + sqlx::query!( + "DELETE FROM remote_deploy_token WHERE workspace_id = $1 AND email = $2", + w_id, + row.email + ) + .execute(&mut *conn) + .await?; + } + } + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn forwarded_url_refuses_paths_the_parser_rewrites() { + let target = RemoteDeployTarget { + base_url: "https://prod.example.com/windmill".to_string(), + workspace_id: "prod".to_string(), + }; + let prefix = "/api/w/dev/remote_deploy/proxy/someKey"; + let url = forwarded_url( + &target, + &format!("{prefix}/scripts/get/p/f/team/my%20script"), + Some("with_starred_info=true"), + ) + .unwrap(); + assert_eq!( + url.as_str(), + "https://prod.example.com/windmill/api/w/prod/scripts/get/p/f/team/my%20script?with_starred_info=true" + ); + for path in [ + format!("{prefix}/../../users/list"), + format!("{prefix}/%2e%2e/%2e%2e/users/list"), + format!("{prefix}/jobs\\run_wait_result\\p/f/team/action"), + prefix.to_string(), + ] { + assert!( + forwarded_url(&target, &path, None).is_err(), + "{path} should be refused" + ); + } + } + + #[test] + fn redacted_uri_masks_the_proxy_key() { + let proxied: axum::http::Uri = "/api/w/dev/remote_deploy/proxy/secretKey/flows/get/f/a?x=1" + .parse() + .unwrap(); + assert_eq!( + RedactedUri(&proxied).to_string(), + "/api/w/dev/remote_deploy/proxy/***/flows/get/f/a?x=1" + ); + let other: axum::http::Uri = "/api/w/dev/flows/list?x=1".parse().unwrap(); + assert_eq!(RedactedUri(&other).to_string(), "/api/w/dev/flows/list?x=1"); + } + + #[test] + fn target_normalization() { + let target = RemoteDeployTarget { + base_url: " https://prod.example.com/ ".to_string(), + workspace_id: " prod ".to_string(), + } + .normalized() + .unwrap(); + assert_eq!(target.base_url, "https://prod.example.com"); + assert_eq!( + target.api_url("flows/create"), + "https://prod.example.com/api/w/prod/flows/create" + ); + + for (base_url, workspace_id) in [ + ("ftp://prod.example.com", "prod"), + ("https://user:pass@prod.example.com", "prod"), + ("https://prod.example.com?token=x", "prod"), + ("https://prod.example.com", "prod/../admins"), + ] { + assert!( + RemoteDeployTarget { + base_url: base_url.to_string(), + workspace_id: workspace_id.to_string(), + } + .normalized() + .is_err(), + "{base_url} {workspace_id} should be refused" + ); + } + } +} diff --git a/backend/windmill-api-workspaces/src/workspaces.rs b/backend/windmill-api-workspaces/src/workspaces.rs index 879babff4f..8ebd4eeb26 100644 --- a/backend/windmill-api-workspaces/src/workspaces.rs +++ b/backend/windmill-api-workspaces/src/workspaces.rs @@ -12,6 +12,7 @@ use windmill_api_auth::{ }; use windmill_api_users::users::WorkspaceInvite; use windmill_common::email_oss::send_email_if_possible; +use windmill_common::ssrf::{validate_webhook_url, webhook_ssrf_error_message}; use windmill_common::usernames::{get_instance_username_or_create_pending, VALID_USERNAME}; use windmill_common::webhook::WebhookShared; use windmill_common::{BASE_URL, DB}; @@ -144,6 +145,7 @@ pub fn workspaced_service() -> Router { ) .merge(crate::datatable_migrations::routes()) .merge(crate::datatable_permissions::routes()) + .merge(crate::datatable_acl::routes()) .route("/git_sync_enabled", get(get_git_sync_enabled)) .route("/git_sync_deploy_mode", get(get_git_sync_deploy_mode)) .route("/edit_git_sync_config", post(edit_git_sync_config)) @@ -500,8 +502,8 @@ struct CreateWorkspaceFork { id: String, name: String, color: Option, - /// Datatable names that were forked. For each, the backend will update the - /// forked workspace's datatable config to point to the new database. + /// Data tables the fork gets a copy of rather than a pointer at the parent's. For each, the + /// fork's entry is pointed at the new database. #[serde(default)] forked_datatables: Vec, /// Lakes the user explicitly chose to SHARE with the parent (the fork then reads and @@ -533,6 +535,11 @@ struct CreateWorkspaceFork { struct ForkedDatatableInfo { name: String, new_dbname: String, + /// What this request copies into `new_dbname`, which it creates. Optional only so that the + /// entry older CLIs send — naming a database they created and filled themselves — is refused + /// with a message rather than a parse error. + #[serde(default)] + fork_behavior: Option, } #[derive(Deserialize)] @@ -1917,6 +1924,16 @@ async fn edit_webhook( )); } + // An empty URL is stored as-is and means "no webhook" to the sender. + if let Some(webhook) = ew.webhook.as_deref().filter(|w| !w.is_empty()) { + validate_webhook_url(webhook).await.map_err(|e| { + Error::BadRequest(format!( + "Webhook URL is not allowed: {}", + webhook_ssrf_error_message(&e) + )) + })?; + } + let mut tx = db.begin().await?; if let Some(webhook) = &ew.webhook { @@ -2232,8 +2249,8 @@ async fn list_datatables( name, resource_type: database.resource_type.as_ref().to_string(), resource_path: database.resource_path.clone(), - governing_workspace_id: (governing.workspace_id != w_id) - .then(|| governing.workspace_id.clone()), + governing_workspace_id: (governing.governing_workspace_id() != w_id) + .then(|| governing.governing_workspace_id().to_string()), permissioned: governing.datatable.permissions.is_some(), }); } @@ -2272,6 +2289,25 @@ struct DataTableTables { schemas: TableListMap, #[serde(skip_serializing_if = "Option::is_none")] error: Option, + /// On the instance database: the only kind that can be under roles or have its access edited. + instance: bool, + permissioned: bool, + /// The roles this caller may connect as, by name; empty when not under roles. + usable_roles: Vec, + default_role: String, + /// What the role the listing connected as may create. + can_create_schema: bool, + creatable_schemas: Vec, +} + +#[derive(Deserialize)] +struct ListDataTableTablesQuery { + /// List only this data table: each entry opens a connection to its database. + datatable_name: Option, + /// The data table `role` applies to. Every other one is listed as its default role, since a + /// role name means nothing outside the data table it belongs to. + role_for: Option, + role: Option, } #[derive(Deserialize)] @@ -2279,6 +2315,7 @@ struct GetDataTableSchemaQuery { datatable_name: String, schema_name: String, table_name: String, + role: Option, } #[derive(Serialize, Debug)] @@ -2446,25 +2483,89 @@ async fn list_datatable_tables( authed: ApiAuthed, Extension(db): Extension, Path(w_id): Path, + Query(query): Query, ) -> JsonResult> { - let datatable_names = list_datatable_names(&db, &w_id).await?; + if query.role.is_some() && query.role_for.is_none() { + return Err(Error::BadRequest( + "`role` needs `role_for`, the data table it is a role of".to_string(), + )); + } + if let (Some(only), Some(role_for)) = + (query.datatable_name.as_deref(), query.role_for.as_deref()) + { + if only != role_for { + return Err(Error::BadRequest(format!( + "`role_for` names '{role_for}', which `datatable_name` leaves out of the listing" + ))); + } + } + let mut datatable_names = list_datatable_names(&db, &w_id).await?; + for named in [query.role_for.as_deref(), query.datatable_name.as_deref()] + .into_iter() + .flatten() + { + if !datatable_names.iter().any(|n| n == named) { + return Err(Error::NotFound(format!( + "No data table named '{named}' in this workspace" + ))); + } + } + if let Some(only) = query.datatable_name.as_deref() { + datatable_names.retain(|n| n == only); + } let mut results = Vec::new(); for datatable_name in datatable_names { - let tables = match get_datatable_tables(&db, &authed, &w_id, &datatable_name).await { - Ok(schemas) => DataTableTables { datatable_name, schemas, error: None }, - Err(e) => DataTableTables { - datatable_name, - schemas: HashMap::new(), - error: Some(e.to_string()), - }, - }; - results.push(tables); + let role = query + .role + .as_deref() + .filter(|_| query.role_for.as_deref() == Some(datatable_name.as_str())); + results.push(list_one_datatable_tables(&db, &authed, &w_id, datatable_name, role).await); } Ok(Json(results)) } +async fn list_one_datatable_tables( + db: &DB, + authed: &ApiAuthed, + w_id: &str, + datatable_name: String, + role: Option<&str>, +) -> DataTableTables { + let mut entry = DataTableTables { + datatable_name, + schemas: HashMap::new(), + error: None, + instance: false, + permissioned: false, + usable_roles: vec![], + default_role: windmill_common::datatable_roles::ADMIN_DATATABLE_ROLE.to_string(), + can_create_schema: false, + creatable_schemas: vec![], + }; + let result: Result<()> = async { + let governing = resolve_governing_datatable(db, w_id, &entry.datatable_name).await?; + entry.instance = governing.is_instance(); + let usable = + crate::datatable_permissions_oss::usable_datatable_roles(db, authed, w_id, &governing) + .await?; + entry.permissioned = usable.permissioned; + entry.usable_roles = usable.roles; + entry.default_role = usable.default_role; + let listing = get_datatable_tables(db, authed, w_id, &entry.datatable_name, role).await?; + entry.schemas = listing.schemas; + entry.can_create_schema = listing.can_create_schema; + entry.creatable_schemas = listing.creatable_schemas; + Ok(()) + } + .await; + if let Err(e) = result { + entry.error = Some(e.to_string()); + } + entry +} + async fn get_datatable_table_schema( authed: ApiAuthed, Extension(db): Extension, @@ -2478,6 +2579,7 @@ async fn get_datatable_table_schema( &query.datatable_name, &query.schema_name, &query.table_name, + query.role.as_deref(), ) .await?; @@ -2516,14 +2618,13 @@ async fn resolve_datatable_pg_as_caller( authed: &ApiAuthed, w_id: &str, datatable_name: &str, + role: Option<&str>, ) -> Result { let db_resource = get_datatable_resource_from_db( db, w_id, datatable_name, - // The data table's default role. Browsing has no way to name another one yet; when the - // database manager grows a role picker it passes the pick through here. - None, + role, DatatableAccess::Authed(authed.to_authed_ref()), ) .await?; @@ -2537,7 +2638,7 @@ async fn get_datatable_schema( w_id: &str, datatable_name: &str, ) -> Result { - let pg_db = resolve_datatable_pg_as_caller(db, authed, w_id, datatable_name).await?; + let pg_db = resolve_datatable_pg_as_caller(db, authed, w_id, datatable_name, None).await?; // Connect to the datatable database let (client, connection) = pg_db.connect(Some(db)).await?; @@ -2625,13 +2726,20 @@ async fn get_datatable_schema( Ok(schema_map) } +struct DatatableTableListing { + schemas: TableListMap, + can_create_schema: bool, + creatable_schemas: Vec, +} + async fn get_datatable_tables( db: &DB, authed: &ApiAuthed, w_id: &str, datatable_name: &str, -) -> Result { - let pg_db = resolve_datatable_pg_as_caller(db, authed, w_id, datatable_name).await?; + role: Option<&str>, +) -> Result { + let pg_db = resolve_datatable_pg_as_caller(db, authed, w_id, datatable_name, role).await?; let (client, connection) = pg_db.connect(Some(db)).await?; tokio::spawn(async move { @@ -2643,7 +2751,7 @@ async fn get_datatable_tables( let schema_rows = client .query( r#" - SELECT nspname::text AS schema_name + SELECT nspname::text AS schema_name, has_schema_privilege(oid, 'CREATE') AS can_create FROM pg_namespace WHERE nspname NOT IN ('information_schema', 'pg_toast', 'pg_catalog') AND nspname NOT LIKE 'pg_%' @@ -2657,11 +2765,29 @@ async fn get_datatable_tables( Error::internal_err(format!("Failed to query schemas: {}", pg_error_message(&e))) })?; + let can_create_schema: bool = client + .query_one( + "SELECT has_database_privilege(current_database(), 'CREATE')", + &[], + ) + .await + .map_err(|e| { + Error::internal_err(format!( + "Failed to read database privileges: {}", + pg_error_message(&e) + )) + })? + .get(0); + let mut table_map: TableListMap = HashMap::new(); + let mut creatable_schemas = Vec::new(); let schema_names: Vec = schema_rows .iter() .map(|row| { let name: String = row.get(0); + if row.get::<_, bool>(1) { + creatable_schemas.push(name.clone()); + } table_map.entry(name.clone()).or_default(); name }) @@ -2691,7 +2817,7 @@ async fn get_datatable_tables( table_map.entry(table_schema).or_default().push(table_name); } - Ok(table_map) + Ok(DatatableTableListing { schemas: table_map, can_create_schema, creatable_schemas }) } async fn get_datatable_table_columns( @@ -2701,6 +2827,7 @@ async fn get_datatable_table_columns( datatable_name: &str, schema_name: &str, table_name: &str, + role: Option<&str>, ) -> Result { if is_system_pg_schema(schema_name) { return Err(Error::BadRequest(format!( @@ -2709,7 +2836,7 @@ async fn get_datatable_table_columns( ))); } - let pg_db = resolve_datatable_pg_as_caller(db, authed, w_id, datatable_name).await?; + let pg_db = resolve_datatable_pg_as_caller(db, authed, w_id, datatable_name, role).await?; let (client, connection) = pg_db.connect(Some(db)).await?; tokio::spawn(async move { @@ -2805,6 +2932,18 @@ fn truncate_column_default(default: String) -> String { mod tests { use super::*; + #[test] + fn a_dev_workspace_copies_into_the_fork_database_namespace() { + assert_eq!( + forked_datatable_dbname("wm-fork-my-fork", "main"), + "wm_fork_my_fork__main" + ); + assert_eq!( + forked_datatable_dbname("my-dev", "main"), + "wm_fork_my_dev__main" + ); + } + /// The header of a pg_dump, followed by an object whose body also holds a `SET`. const DUMP: &str = "--\n\ -- PostgreSQL database dump\n\ @@ -3273,7 +3412,10 @@ async fn server_setting_names(pg_db: &PgDatabase) -> Result> { /// so a dump that breaks partway through imports partially and reads as a success. /// ON_ERROR_STOP surfaces the failure and --single-transaction makes the restore /// all-or-nothing, leaving the target as it was and the import retryable. -async fn pg_import_dump(target_db: &PgDatabase, dump_file: &DumpFile) -> Result<()> { +/// +/// Authorization: none. Writes into `target_db` with the credentials it carries; callers MUST have +/// authorized writing to that database. +pub(crate) async fn pg_import_dump(target_db: &PgDatabase, dump_file: &DumpFile) -> Result<()> { let supported_settings = server_setting_names(target_db).await?; comment_out_unsupported_settings(dump_file, &supported_settings).await?; @@ -3322,7 +3464,7 @@ async fn create_pg_database( // exists rather than leaving an empty registered `wm_fork_…` behind. if let Some(reference) = req.source.strip_prefix("datatable://") { let (name, _) = parse_datatable_ref_for(&db, &w_id, reference).await?; - ensure_datatable_is_clonable(&db, &w_id, &name).await?; + ensure_copied_without_roles(&ensure_datatable_is_clonable(&db, &w_id, &name).await?)?; } // Non-superadmin: restrict dbname to wm_fork_ prefix @@ -3341,50 +3483,62 @@ async fn create_pg_database( } else { let source_pg = resolve_pg_source_checked(&db, &user_db, &authed, &w_id, &req.source).await?; - let (client, connection) = source_pg.connect(Some(&db)).await?; - let join_handle = tokio::spawn(async move { connection.await }); - - let row = client - .query_one( - "SELECT EXISTS (SELECT 1 FROM pg_catalog.pg_database WHERE datname = $1)", - &[&req.target_dbname], - ) - .await - .map_err(|e| { - Error::internal_err(format!( - "Failed to check database existence: {}", - pg_error_message(&e) - )) - })?; - let db_exists: bool = row.get(0); - - if db_exists { - drop(client); - let _ = windmill_common::shutdown_pg_connection(join_handle).await; - return Err(Error::BadRequest(format!( - "Database '{}' already exists on the resource server", - req.target_dbname - ))); - } - - client - .execute(&format!("CREATE DATABASE \"{}\"", &req.target_dbname), &[]) - .await - .map_err(|e| { - Error::internal_err(format!( - "Failed to create database '{}': {}", - req.target_dbname, - pg_error_message(&e) - )) - })?; - - drop(client); - windmill_common::shutdown_pg_connection(join_handle).await?; + create_database_on_server(&db, &source_pg, &req.target_dbname).await?; } Ok(format!("Created database '{}'", req.target_dbname)) } +/// `CREATE DATABASE` on the server `server` connects to, refusing a name already taken there. +/// +/// Authorization: none. Callers MUST have authorized creating a database on that server — resolved +/// from a data table or resource the caller may administer. +pub(crate) async fn create_database_on_server( + db: &DB, + server: &PgDatabase, + dbname: &str, +) -> Result<()> { + windmill_common::validate_dbname(dbname)?; + let (client, connection) = server.connect(Some(db)).await?; + let join_handle = tokio::spawn(async move { connection.await }); + + let row = client + .query_one( + "SELECT EXISTS (SELECT 1 FROM pg_catalog.pg_database WHERE datname = $1)", + &[&dbname], + ) + .await + .map_err(|e| { + Error::internal_err(format!( + "Failed to check database existence: {}", + pg_error_message(&e) + )) + })?; + let db_exists: bool = row.get(0); + + if db_exists { + drop(client); + let _ = windmill_common::shutdown_pg_connection(join_handle).await; + return Err(Error::BadRequest(format!( + "Database '{dbname}' already exists on the resource server" + ))); + } + + client + .execute(&format!("CREATE DATABASE \"{dbname}\""), &[]) + .await + .map_err(|e| { + Error::internal_err(format!( + "Failed to create database '{dbname}': {}", + pg_error_message(&e) + )) + })?; + + drop(client); + windmill_common::shutdown_pg_connection(join_handle).await?; + Ok(()) +} + #[derive(Deserialize)] struct ImportPgDatabaseRequest { source: String, @@ -3394,48 +3548,22 @@ struct ImportPgDatabaseRequest { fork_behavior: DataTableForkBehavior, } -/// Refuse to copy a data table that is under roles. +/// Whether data table `name` of `w_id` has a shape a copy can be made of, returning what governs +/// it. Checked before anything is created — by the fork request for the copies it makes, and by +/// `create_pg_database` — and again when the fork's entry is written. /// -/// `pg_dump` carries no roles and the import runs with `--no-privileges`, so a clone arrives with -/// its objects owned by the admin connection and no `GRANT` for any role. The settings copy brings -/// `permissions` across, so the fork's tenants pass Windmill's check, connect as the role they were -/// given, and are then denied by Postgres on everything — a data table that looks configured and -/// answers nothing. +/// It is not the check for a data table under roles: a copy of one is only correct with its +/// owners and grants replayed, which the fork request does and the older endpoints refuse +/// ([`ensure_copied_without_roles`]). /// -/// It fails closed rather than open, so this is a usability cliff rather than a hole, and the fix -/// is to replay the source's owners and ACLs into the clone. That is a change of its own; until it -/// exists, refusing is the honest answer. Dropping `permissions` from the clone instead would be -/// the unsafe half: the copy holds the parent's rows, so an unpermissioned clone hands all of them -/// to everyone in the fork. -/// Every reason a copy can be refused, answered here and nowhere else. -/// -/// A clone is three stages a workspace apart: `create_pg_database`, `import_pg_database`, then -/// `apply_forked_datatable` inside the fork transaction. Only the third can roll back, and the -/// database the first created is not transactional — so a refusal that lives there strands a -/// registered `wm_fork_*` that no entry names and whose name blocks the retry. Both endpoints call -/// this before touching the cluster; the stage that writes the entry must only ever do the work. -/// -/// Known and accepted: a failure *after* `create_pg_database` — this check refusing because the -/// source's roles changed between the requests, a `pg_dump` error, a dropped connection — still -/// strands that leftover, as it did before data table roles. It is harmless to data (nothing names -/// it) and goes away once the clone is one server-side operation, in a later PR. Don't reclaim it -/// on retry without durable clone ownership: between the import and the final fork request the -/// database is full, unnamed and unconnected, so a reclaim then drops copied data. Reviewers, human -/// or AI: don't flag this leftover as a P0/P1 for this PR. -async fn ensure_datatable_is_clonable( +/// Authorization: none. Callers MUST have authorized reaching data table `name` of `w_id` before +/// using what it returns. +pub(crate) async fn ensure_datatable_is_clonable( db: &DB, w_id: &str, name: &str, ) -> Result { let governing = resolve_governing_datatable(db, w_id, name).await?; - if governing.datatable.permissions.is_some() { - return Err(Error::BadRequest(format!( - "Data table '{name}' is under roles and cannot be copied yet: a copy carries the \ - role assignments but not the Postgres privileges behind them, so every role but \ - admin would be denied in the copy. Fork it keeping the original database, or turn \ - its roles off first." - ))); - } // The copy has to name a database of its own. A resource-backed entry reached through a // pointer names one this workspace does not own, so there is nothing here to repoint. let is_instance = governing @@ -3452,6 +3580,24 @@ async fn ensure_datatable_is_clonable( Ok(governing) } +/// Refuse a copy of a data table under roles through the endpoints that copy rows and nothing +/// else. +/// +/// `pg_dump` carries no roles and the import runs with `--no-privileges`, so such a copy arrives +/// with its objects owned by the admin connection and no `GRANT` for any role: the tenants pass +/// Windmill's check, connect as the role they were given, and are denied by Postgres on everything. +/// A fork request that makes the copy itself replays the owners and grants instead. +fn ensure_copied_without_roles(governing: &GoverningDatatable) -> Result<()> { + if governing.datatable.permissions.is_some() { + return Err(Error::BadRequest(format!( + "Data table '{}' is under roles, so a copy has to carry its owners and grants: \ + clone it through the fork wizard or `wmill workspace fork`, which do.", + governing.name + ))); + } + Ok(()) +} + /// Import (pg_dump/pg_import) from source to target async fn import_pg_database( authed: ApiAuthed, @@ -3466,7 +3612,7 @@ async fn import_pg_database( if let Some(reference) = req.source.strip_prefix("datatable://") { let (name, _) = parse_datatable_ref_for(&db, &w_id, reference).await?; - ensure_datatable_is_clonable(&db, &w_id, &name).await?; + ensure_copied_without_roles(&ensure_datatable_is_clonable(&db, &w_id, &name).await?)?; } if req.fork_behavior == DataTableForkBehavior::SchemaAndData { @@ -3587,6 +3733,15 @@ async fn edit_ducklake_config( } let mut tx = db.begin().await?; + windmill_common::lock_instance_databases( + &mut tx, + new_config.settings.ducklakes.values().filter_map(|lake| { + (lake.catalog.resource_type + == windmill_common::workspaces::DucklakeCatalogResourceType::Instance) + .then_some(lake.catalog.resource_path.as_str()) + }), + ) + .await?; let args_for_audit = format!("{:?}", new_config.settings); audit_log( @@ -3688,6 +3843,16 @@ async fn edit_datatable_config( let is_superadmin = require_super_admin(&db, &authed).await.is_ok(); let mut tx = db.begin().await?; + windmill_common::lock_instance_databases( + &mut tx, + new_config.settings.datatables.values().filter_map(|dt| { + dt.database + .as_ref() + .filter(|d| d.resource_type == DataTableCatalogResourceType::Instance) + .map(|d| d.resource_path.as_str()) + }), + ) + .await?; // Read under the row lock this transaction will write with. `permissions`, `reference` and // `forked_from` are carried across from what this read returns, so a permissions save @@ -3813,12 +3978,28 @@ async fn edit_datatable_config( // hand the fork the database outright. `forked_from` is the clone stamp the fork flow // writes: whether an entry has one is carried the same way, since it is what marks the // database droppable, but the schema baseline inside it is the diff view's to advance. + // `governed_by` is a clone's `reference` for its roles, and clearing it would hand the + // fork the copied rows the same way. dt.permissions = old.and_then(|old| old.permissions.clone()); dt.reference = old.and_then(|old| old.reference.clone()); + dt.governed_by = old.and_then(|old| old.governed_by.clone()); dt.forked_from = match old.and_then(|old| old.forked_from.as_ref()) { Some(stored) => Some(dt.forked_from.take().unwrap_or_else(|| stored.clone())), None => None, }; + // A clone's roles and grants were replayed into the database it was copied into, and hold + // for that database alone: whoever saves, it stays where it is. + if dt.governed_by.is_some() + && !crate::datatable_clone::same_database( + &dt.database, + &old.and_then(|old| old.database.clone()), + ) + { + return Err(Error::BadRequest(format!( + "Data table '{name}' is a clone taking its roles from the data table it was copied \ + from, and its grants hold for its own database only: it cannot be moved." + ))); + } // Carrying the block onto a resource-backed entry would produce a data table the chokepoint // refuses on every job — a save that succeeds and breaks everything afterwards. Refuse it // instead: turning roles off first is one step, and it keeps discarding an access decision @@ -3857,9 +4038,18 @@ async fn edit_datatable_config( // workspace does not own. Pointing an entry at another workspace's data table is not checked // here because it cannot be requested at all: `reference` is overwritten from the stored entry // above, for every caller. + // + // Compared against the entry the carried fields came from, not the one stored under the same + // name: otherwise swapping two names keeps each database in place while moving a clone's + // `governed_by` off the copy it governs. if !is_superadmin { for (name, dt) in new_config.settings.datatables.iter() { - let old_dt = old_datatables.get(name); + let old_dt = old_datatables.get( + rename_src + .get(name.as_str()) + .copied() + .unwrap_or(name.as_str()), + ); if dt .database .as_ref() @@ -3899,7 +4089,8 @@ async fn edit_datatable_config( .settings .datatables .iter() - .filter(|(_, dt)| dt.permissions.is_none()) + // A clone carries its roles through `governed_by` rather than `permissions`. + .filter(|(_, dt)| dt.permissions.is_none() && dt.governed_by.is_none()) .filter_map(|(name, dt)| { let db = dt .database @@ -3932,7 +4123,7 @@ async fn edit_datatable_config( sqlx::query_scalar( "SELECT DISTINCT dt.value->'database'->>'resource_path' FROM workspace_settings ws CROSS JOIN LATERAL jsonb_each(COALESCE(ws.datatable->'datatables', '{}'::jsonb)) dt - WHERE ws.workspace_id <> $1 AND dt.value ? 'permissions' + WHERE ws.workspace_id <> $1 AND (dt.value ? 'permissions' OR dt.value ? 'governed_by') AND dt.value->'database'->>'resource_type' = 'instance'", ) .bind(&w_id) @@ -3941,7 +4132,7 @@ async fn edit_datatable_config( }; for (name, dbname) in newly_pointed { let governed_here = old_datatables.values().any(|old| { - old.permissions.is_some() + (old.permissions.is_some() || old.governed_by.is_some()) && old.database.as_ref().is_some_and(|d| { d.resource_type == DataTableCatalogResourceType::Instance && d.resource_path == dbname @@ -4001,8 +4192,11 @@ async fn edit_datatable_config( r#"SELECT ws.workspace_id AS "workspace_id!", dt.key AS "datatable!" FROM workspace_settings ws CROSS JOIN LATERAL jsonb_each(COALESCE(ws.datatable->'datatables', '{}'::jsonb)) dt - WHERE dt.value->'reference'->>'workspace_id' = $1 - AND dt.value->'reference'->>'datatable' = $2"#, + WHERE EXISTS ( + SELECT 1 FROM (VALUES ('reference'), ('governed_by')) k(link) + WHERE dt.value->k.link->>'workspace_id' = $1 + AND dt.value->k.link->>'datatable' = $2 + )"#, &w_id, name, ) @@ -5777,6 +5971,13 @@ async fn set_encryption_key( &new_encryption_key, ) .await?; + crate::remote_deploy::reencrypt_tokens( + &mut tx, + &w_id, + &previous_encryption_key, + &new_encryption_key, + ) + .await?; tx.commit().await?; @@ -7961,14 +8162,32 @@ async fn clone_drafts( // filtered like `clone_scripts`: the address the draft still carries re-derives the // clone's own principal at deploy time, which is the more accurate answer of the two. sqlx::query!( - "INSERT INTO draft (workspace_id, path, typ, value, created_at, email) + // A script hash is content-addressed and copied as-is, so a script draft's base + // still names a version the clone has. `clone_flows` / `clone_apps` mint new ids, + // so those drafts arrive with no base (staleness falls back to the timestamps), + // lineage field included, or the next autosave would re-derive the source id. + // + // `clean` is `strip_json_nul`'s parity rule in SQL, so a pre-sanitizer U+0000 + // escape cannot abort the clone on `to_jsonb` or arrive with its principal + // unstripped: escaped backslashes park on chr(1) (lossless, a `json` value's text + // cannot hold a raw control byte) so only a real NUL is removed, and chr(92) + // spells the backslash so no escape sequence reaches this source file. + r#"INSERT INTO draft (workspace_id, path, typ, value, created_at, email, base) SELECT $2, path, typ, - CASE WHEN typ IN ('script', 'flow') - THEN to_json(to_jsonb(value) - 'on_behalf_of') - ELSE value END, - created_at, email - FROM draft - WHERE workspace_id = $1 AND (email = $3 OR email IS NULL)", + to_json( + CASE WHEN typ IN ('script', 'flow') THEN clean - 'on_behalf_of' ELSE clean END + - CASE WHEN typ = 'flow' THEN 'version_id' + WHEN typ IN ('app', 'raw_app') THEN 'parent_version' + ELSE '' END + ), + created_at, email, + CASE WHEN typ = 'script' THEN base END + FROM ( + SELECT d.path, d.typ, d.created_at, d.email, d.base, + replace(replace(replace(d.value::text, chr(92) || chr(92), chr(1)), chr(92) || 'u0000', ''), chr(1), chr(92) || chr(92))::jsonb AS clean + FROM draft d + WHERE d.workspace_id = $1 AND (d.email = $3 OR d.email IS NULL) + ) s"#, source_workspace_id, target_workspace_id, authed_email, @@ -8112,6 +8331,7 @@ async fn create_workspace_fork_branch( // dangling branch on the synced repos. check_fork_w_id_conflict(&db, &nw.id).await?; purge_stale_fork_diff_state(&db, &nw.id).await?; + validate_forked_datatables(&db, &authed, &w_id, &nw).await?; Ok(Json( handle_fork_branch_creation(&authed.email, &authed.username, &db, &w_id, &nw.id).await?, @@ -8151,8 +8371,9 @@ async fn snapshot_datatable_schema( /// a fork admin could then edit to widen their own access to it. A pointer has nothing local to /// edit: the parent's entry stays the only place the decision lives. /// -/// The cloned data tables are skipped: they own a fresh database of their own, and they keep the -/// copied `permissions` as their starting point, which they then govern. +/// The cloned data tables are skipped: they own a fresh database of their own, and +/// `apply_forked_datatable` has already dropped their copied `permissions` — for a clone of a data +/// table under roles, in favor of `governed_by`. async fn point_kept_datatables_at_parent( tx: &mut Transaction<'_, Postgres>, parent_w_id: &str, @@ -8207,6 +8428,7 @@ async fn point_kept_datatables_at_parent( workspace_id: parent_w_id.to_string(), datatable: name.clone(), }), + governed_by: None, forked_from: None, migrations_enabled: dt.migrations_enabled, permissions: None, @@ -8227,7 +8449,8 @@ async fn point_kept_datatables_at_parent( Ok(()) } -/// Move every pointer in any workspace that names `(w_id, from)` to `(w_id, to)`. +/// Move every pointer, and every clone's `governed_by`, in any workspace that names `(w_id, from)` +/// to `(w_id, to)`. /// /// `EXISTS` rather than a `LIKE` over the whole document: the update rewrites the row, so matching /// every workspace that holds any pointer would rewrite rows to a byte-identical value and hold an @@ -8243,17 +8466,22 @@ async fn repoint_datatable_references( SET datatable = ( SELECT jsonb_set(ws.datatable, '{datatables}', jsonb_object_agg( dt.key, - CASE WHEN dt.value->'reference'->>'workspace_id' = $1 - AND dt.value->'reference'->>'datatable' = $2 - THEN jsonb_set(dt.value, '{reference,datatable}', to_jsonb($3::text)) - ELSE dt.value END + (SELECT CASE WHEN r.v->'governed_by'->>'workspace_id' = $1 + AND r.v->'governed_by'->>'datatable' = $2 + THEN jsonb_set(r.v, '{governed_by,datatable}', to_jsonb($3::text)) + ELSE r.v END + FROM (SELECT CASE WHEN dt.value->'reference'->>'workspace_id' = $1 + AND dt.value->'reference'->>'datatable' = $2 + THEN jsonb_set(dt.value, '{reference,datatable}', to_jsonb($3::text)) + ELSE dt.value END AS v) r) )) FROM jsonb_each(ws.datatable->'datatables') dt ) WHERE EXISTS ( - SELECT 1 FROM jsonb_each(COALESCE(ws.datatable->'datatables', '{}'::jsonb)) d - WHERE d.value->'reference'->>'workspace_id' = $1 - AND d.value->'reference'->>'datatable' = $2 + SELECT 1 FROM jsonb_each(COALESCE(ws.datatable->'datatables', '{}'::jsonb)) d, + LATERAL (VALUES ('reference'), ('governed_by')) k(link) + WHERE d.value->k.link->>'workspace_id' = $1 + AND d.value->k.link->>'datatable' = $2 )"#, w_id, from, @@ -8271,17 +8499,8 @@ async fn apply_forked_datatable( parent_w_id: &str, forked_w_id: &str, fdt: &ForkedDatatableInfo, + copy: &crate::datatable_clone::MadeCopy, ) -> Result<()> { - // Cloning reads the parent's whole schema as admin and hands the copy to the fork, so it is - // for the workspace that governs the data table — a fork can use one, never duplicate it. - windmill_common::workspaces::ensure_datatable_admin_access( - db, - parent_w_id, - &fdt.name, - &DatatableAccess::Authed(authed.to_authed_ref()), - ) - .await?; - let governing = ensure_datatable_is_clonable(db, parent_w_id, &fdt.name).await?; windmill_common::validate_dbname(&fdt.new_dbname)?; if !fdt.new_dbname.starts_with("wm_fork_") { return Err(Error::BadRequest(format!( @@ -8289,6 +8508,52 @@ async fn apply_forked_datatable( fdt.new_dbname ))); } + // Held until the fork commits, as the permissions save holds them: the source moved onto + // another database, or put under roles or taken off them, since its copy was made would link + // the copy to roles its grants were not replayed for. + let locked = crate::datatable_clone::lock_settings_rows(tx, parent_w_id, &fdt.name).await?; + let governing = ensure_datatable_is_clonable(db, parent_w_id, &fdt.name).await?; + let under_roles = governing.datatable.permissions.is_some(); + let unchanged = crate::datatable_clone::same_database( + &governing.datatable.database, + &Some(copy.source_database.clone()), + ) && copy.replayed == under_roles + && crate::datatable_clone::lock_settings_rows(tx, parent_w_id, &fdt.name).await? == locked; + if !unchanged { + return Err(Error::BadRequest(format!( + "Data table '{}' changed while it was being copied for this fork — its database, or \ + whether it is under roles. Create the fork again.", + fdt.name + ))); + } + // The fork keeps a snapshot of the source's schema, which under roles is only for those who may + // connect as one of them — as the copy itself was. + crate::datatable_permissions::ensure_reaches_governing_datatable( + db, + parent_w_id, + &fdt.name, + &governing, + authed, + ) + .await?; + // Under roles, the copy holds rows the source's roles decide who reaches, so that entry keeps + // deciding. Settled from the source as it resolves now: the settings clone may have handed the + // fork a pointer, or a clone of its own. A copy of a data table without roles is the fork's, + // as it was before roles existed: everyone reached all of it already. + let governed_by = governing + .datatable + .permissions + .is_some() + .then(|| { + serde_json::to_value(governing.governor.clone().unwrap_or_else(|| { + windmill_common::workspaces::DataTableReference { + workspace_id: governing.workspace_id.clone(), + datatable: governing.name.clone(), + } + })) + }) + .transpose() + .map_err(|e| Error::internal_err(format!("serializing a clone's governor: {e}")))?; // Snapshot the schema from the source (parent) datatable let schema = snapshot_datatable_schema(db, parent_w_id, &fdt.name).await?; @@ -8334,33 +8599,42 @@ async fn apply_forked_datatable( "resource_type": "instance", "resource_path": &fdt.new_dbname, }); - sqlx::query!( + sqlx::query( r#"UPDATE workspace_settings SET datatable = jsonb_set( - jsonb_set( - datatable #- ARRAY['datatables', $2, 'reference'], - ARRAY['datatables', $2, 'database'], $3::jsonb), - ARRAY['datatables', $2, 'forked_from'], $4::jsonb - ) + datatable #- ARRAY['datatables', $2, 'reference'], + ARRAY['datatables', $2, 'database'], $3::jsonb) WHERE workspace_id = $1"#, - forked_w_id, - &fdt.name, - new_database, - forked_from, ) + .bind(forked_w_id) + .bind(&fdt.name) + .bind(new_database) .execute(&mut **tx) .await?; } else { - // Resource: update the resource's dbname and mark as ws_specific + // Resource: point it at the copy and mark it ws_specific. What the settings clone carried is + // the source's resource and variables as they are now, which an edit made while the copy + // was being made can have changed, and changed back: the clone has to be what the copy's + // connection was resolved from. let resource_path = &database.resource_path; - sqlx::query!( + let cloned = + crate::datatable_clone::connection_snapshot(db, &mut **tx, forked_w_id, resource_path) + .await?; + if copy.connection.as_ref() != Some(&cloned) { + return Err(Error::BadRequest(format!( + "The resource of data table '{}' changed while it was being copied for this \ + fork. Create the fork again.", + fdt.name + ))); + } + sqlx::query( r#"UPDATE resource SET value = jsonb_set(value, '{dbname}', to_jsonb($3::text)) WHERE workspace_id = $1 AND path = $2"#, - forked_w_id, - resource_path, - &fdt.new_dbname, ) + .bind(forked_w_id) + .bind(resource_path) + .bind(&fdt.new_dbname) .execute(&mut **tx) .await?; @@ -8371,20 +8645,31 @@ async fn apply_forked_datatable( ) .execute(&mut **tx) .await?; - - // Set forked_from on the datatable config - sqlx::query!( - r#"UPDATE workspace_settings - SET datatable = jsonb_set(datatable, ARRAY['datatables', $2, 'forked_from'], $3::jsonb) - WHERE workspace_id = $1"#, - forked_w_id, - &fdt.name, - forked_from, - ) - .execute(&mut **tx) - .await?; } + // The settings clone copied the source's `permissions` along, and a clone of a clone its + // `governed_by`: a clone keeps no `permissions` of its own, and a link only under roles. + sqlx::query( + r#"UPDATE workspace_settings + SET datatable = jsonb_set( + CASE WHEN $3::jsonb IS NULL + THEN datatable #- ARRAY['datatables', $2, 'permissions'] + #- ARRAY['datatables', $2, 'governed_by'] + ELSE jsonb_set( + datatable #- ARRAY['datatables', $2, 'permissions'], + ARRAY['datatables', $2, 'governed_by'], $3::jsonb) + END, + ARRAY['datatables', $2, 'forked_from'], $4::jsonb + ) + WHERE workspace_id = $1"#, + ) + .bind(forked_w_id) + .bind(&fdt.name) + .bind(governed_by) + .bind(forked_from) + .execute(&mut **tx) + .await?; + Ok(()) } @@ -8719,7 +9004,169 @@ async fn create_workspace_fork( ensure_no_existing_dev_workspace(&db, &parent_workspace_id).await?; } + // Refused here, before any database exists; `make_copies` checks again under the locks. + validate_forked_datatables(&db, &authed, &parent_workspace_id, &nw).await?; + + // Detached from the request: a client that goes away while the copies are made must still end + // with the fork created or no copy left behind, and dropping the handler's future would skip + // that cleanup. + tokio::spawn(async move { + let fork_id = nw.id.clone(); + let copies = crate::datatable_clone::make_copies( + &db, + &authed, + &parent_workspace_id, + ©_requests(&nw.forked_datatables), + ) + .await?; + let replayed: Vec = copies + .iter() + .filter(|c| c.replayed) + .map(|c| c.behavior) + .collect(); + match write_workspace_fork( + db.clone(), + authed, + parent_workspace_id, + nw, + dev_workspace_label, + &copies, + ) + .await + { + Ok(message) => { + for behavior in replayed { + windmill_common::feature_usage::log_feature_usage( + "datatable", + "clone_replayed", + match behavior { + DataTableForkBehavior::SchemaOnly => "schema_only", + _ => "schema_and_data", + }, + ); + } + Ok(message) + } + // A commit whose acknowledgement was lost can still have committed, and a concurrent + // request for the same id can have: once the write has settled, the cleanup keeps + // whichever copies a committed fork names, and drops the rest. + Err(e) => match wait_for_fork_write(&db, &fork_id).await { + Ok(()) => Err(crate::datatable_clone::drop_copies_after(&db, copies, e).await), + Err(settle) => { + tracing::error!( + "Could not tell whether fork '{fork_id}' was created, so its copies were \ + kept: {settle}" + ); + Err(e) + } + }, + } + }) + .await + .map_err(|e| Error::internal_err(format!("Creating the fork stopped unexpectedly: {e}")))? +} + +/// The database a data table of the fork or dev workspace `fork_id` is copied into, as the wizard +/// and the CLI derive it. A dev workspace's id has no `wm-fork-` prefix, and its copies are dropped +/// by the same `wm_fork_` rule as a fork's. Dev workspace `x` and fork `wm-fork-x` share a name, as +/// their branches do: the second to copy a data table of that name is refused at `CREATE`. +fn forked_datatable_dbname(fork_id: &str, datatable: &str) -> String { + let suffix = fork_id + .strip_prefix(windmill_common::workspaces::WM_FORK_PREFIX) + .unwrap_or(fork_id); + format!("wm_fork_{}__{datatable}", suffix.replace('-', "_")) +} + +/// Wait until no transaction writing fork `fork_id` is still running: the lock +/// `write_workspace_fork` holds until its transaction ends is taken and released. +async fn wait_for_fork_write(db: &DB, fork_id: &str) -> Result<()> { + let mut tx = db.begin().await?; + sqlx::query("SELECT pg_advisory_xact_lock(hashtext('fork:' || $1))") + .bind(fork_id) + .execute(&mut *tx) + .await?; + tx.commit().await?; + Ok(()) +} + +/// Everything about a fork's data tables that can be refused before a git branch or a database is +/// created, and that the fork request re-checks. +/// +/// Every data table is copied by the fork request, into a database it creates: an entry naming a +/// database that already exists could reach a copy made for another fork — one left behind by a +/// deleted fork included — without that copy's governance. +async fn validate_forked_datatables( + db: &DB, + authed: &ApiAuthed, + parent_w_id: &str, + nw: &CreateWorkspaceFork, +) -> Result<()> { + let mut names = HashSet::new(); + for fdt in &nw.forked_datatables { + if fdt.fork_behavior.is_none() { + return Err(Error::BadRequest(format!( + "Data table '{}' names no `fork_behavior`: the fork request makes the copy of each \ + data table itself. Update the Windmill CLI.", + fdt.name + ))); + } + let expected = forked_datatable_dbname(&nw.id, &fdt.name); + if fdt.new_dbname != expected { + return Err(Error::BadRequest(format!( + "Data table '{}' of fork '{}' is copied into database '{expected}', not '{}'", + fdt.name, nw.id, fdt.new_dbname + ))); + } + if !names.insert(fdt.name.as_str()) { + return Err(Error::BadRequest(format!( + "Data table '{}' is named more than once in this fork", + fdt.name + ))); + } + } + crate::datatable_clone::authorize_copies( + db, + authed, + parent_w_id, + ©_requests(&nw.forked_datatables), + ) + .await +} + +/// The copies a fork request asks this request to make. [`validate_forked_datatables`] refuses an +/// entry without a `fork_behavior`. +fn copy_requests( + forked_datatables: &[ForkedDatatableInfo], +) -> Vec> { + forked_datatables + .iter() + .filter_map(|fdt| { + fdt.fork_behavior + .map(|behavior| crate::datatable_clone::CopyRequest { + name: &fdt.name, + dbname: &fdt.new_dbname, + behavior, + }) + }) + .collect() +} + +/// Write the fork, with its data tables pointed at `copies`. +async fn write_workspace_fork( + db: DB, + authed: ApiAuthed, + parent_workspace_id: String, + nw: CreateWorkspaceFork, + dev_workspace_label: Option, + copies: &[crate::datatable_clone::MadeCopy], +) -> Result { let mut tx: Transaction<'_, Postgres> = db.begin().await?; + // Held until this transaction ends: after an error, the copies' cleanup waits on it, so it reads + // whether the fork exists only once a commit still being resolved has settled. + sqlx::query("SELECT pg_advisory_xact_lock(hashtext('fork:' || $1))") + .bind(&nw.id) + .execute(&mut *tx) + .await?; if nw.is_dev_workspace { // The checks above ran outside a transaction, so the parent's eligibility and the chain's @@ -8845,10 +9292,41 @@ async fn create_workspace_fork( repoint_unresolvable_cloned_identities(&mut tx, &forked_id, &authed).await?; + // Held until the fork commits, as settings saves naming these databases hold it: an entry + // saved elsewhere while a copy was being made would reach it without the governance written + // below, and the alias check on that save saw no governed entry yet. + let instance_copies: Vec<&str> = copies + .iter() + .filter(|c| c.source_database.resource_type == DataTableCatalogResourceType::Instance) + .map(|c| c.dbname.as_str()) + .collect(); + windmill_common::lock_instance_databases(&mut tx, instance_copies.iter().copied()).await?; + for dbname in &instance_copies { + let users = windmill_common::instance_database_users(&mut *tx, dbname).await?; + if !users.is_empty() { + return Err(Error::BadRequest(format!( + "Database '{dbname}' copied for this fork is already used by workspaces {}; \ + fork again", + users.join(", ") + ))); + } + } + // Update forked datatable settings to point to new databases for fdt in &nw.forked_datatables { - apply_forked_datatable(&db, &mut tx, &authed, &parent_workspace_id, &forked_id, fdt) - .await?; + let copy = copies.iter().find(|c| c.name == fdt.name).ok_or_else(|| { + Error::internal_err(format!("No copy was made of data table '{}'", fdt.name)) + })?; + apply_forked_datatable( + &db, + &mut tx, + &authed, + &parent_workspace_id, + &forked_id, + fdt, + copy, + ) + .await?; } point_kept_datatables_at_parent( @@ -8907,6 +9385,20 @@ async fn create_workspace_fork( .await?; } + let copied = copies + .iter() + .map(|c| { + format!( + "{} ({})", + c.name, + match c.behavior { + DataTableForkBehavior::SchemaOnly => "schema_only", + _ => "schema_and_data", + } + ) + }) + .collect::>() + .join(", "); audit_log( &mut *tx, &authed, @@ -8914,7 +9406,7 @@ async fn create_workspace_fork( ActionKind::Create, &forked_id, Some(nw.name.as_str()), - None, + (!copied.is_empty()).then(|| [("copied_datatables", copied.as_str())].into()), ) .await?; tx.commit().await?; @@ -9669,13 +10161,25 @@ async fn leave_workspace( &format!("u/{}", authed.username), ) .await?; - sqlx::query!( + let left = sqlx::query!( "DELETE FROM usr WHERE workspace_id = $1 AND email = $2", &w_id, &authed.email ) .execute(&mut *tx) - .await?; + .await? + .rows_affected(); + // A superadmin deploys from workspaces it is no member of; leaving none is no reason to drop + // its connection. + if left > 0 { + sqlx::query!( + "DELETE FROM remote_deploy_token WHERE email = $1 AND workspace_id = $2", + &authed.email, + &w_id + ) + .execute(&mut *tx) + .await?; + } audit_log( &mut *tx, diff --git a/backend/windmill-api-workspaces/src/workspaces_extra.rs b/backend/windmill-api-workspaces/src/workspaces_extra.rs index 45f82227f5..9039bda2ef 100644 --- a/backend/windmill-api-workspaces/src/workspaces_extra.rs +++ b/backend/windmill-api-workspaces/src/workspaces_extra.rs @@ -113,7 +113,7 @@ pub(crate) async fn change_workspace_id( // Duplicate workspace settings (keep copy in old workspace for reference) info!("Duplicating workspace_settings table"); sqlx::query!( - "INSERT INTO workspace_settings (workspace_id, slack_team_id, slack_name, slack_command_script, slack_email, customer_id, plan, webhook, ai_config, large_file_storage, git_sync, default_app, default_scripts, deploy_ui, mute_critical_alerts, color, operator_settings, teams_command_script, teams_team_id, teams_team_name, git_app_installations, git_credentials, ducklake, dbt_warehouses, slack_oauth_client_id, slack_oauth_client_secret, datatable, teams_team_guid, auto_invite, error_handler, success_handler, public_app_execution_limit_per_minute, error_handler_fallback_to_instance_alerts, guest_access_enabled, guest_jwt_public_key, guest_jwt_jwks_url, add_admins_and_developers_to_forks) SELECT $1, slack_team_id, slack_name, slack_command_script, slack_email, customer_id, plan, webhook, ai_config, large_file_storage, git_sync, default_app, default_scripts, deploy_ui, mute_critical_alerts, color, operator_settings, teams_command_script, teams_team_id, teams_team_name, git_app_installations, git_credentials, ducklake, dbt_warehouses, slack_oauth_client_id, slack_oauth_client_secret, datatable, teams_team_guid, auto_invite, error_handler, success_handler, public_app_execution_limit_per_minute, error_handler_fallback_to_instance_alerts, guest_access_enabled, guest_jwt_public_key, guest_jwt_jwks_url, add_admins_and_developers_to_forks FROM workspace_settings WHERE workspace_id = $2", + "INSERT INTO workspace_settings (workspace_id, slack_team_id, slack_name, slack_command_script, slack_email, customer_id, plan, webhook, ai_config, large_file_storage, git_sync, default_app, default_scripts, deploy_ui, mute_critical_alerts, color, operator_settings, teams_command_script, teams_team_id, teams_team_name, git_app_installations, git_credentials, ducklake, dbt_warehouses, slack_oauth_client_id, slack_oauth_client_secret, datatable, teams_team_guid, auto_invite, error_handler, success_handler, public_app_execution_limit_per_minute, error_handler_fallback_to_instance_alerts, guest_access_enabled, guest_jwt_public_key, guest_jwt_jwks_url, add_admins_and_developers_to_forks, remote_deploy_target, remote_deploy_target_changed_at) SELECT $1, slack_team_id, slack_name, slack_command_script, slack_email, customer_id, plan, webhook, ai_config, large_file_storage, git_sync, default_app, default_scripts, deploy_ui, mute_critical_alerts, color, operator_settings, teams_command_script, teams_team_id, teams_team_name, git_app_installations, git_credentials, ducklake, dbt_warehouses, slack_oauth_client_id, slack_oauth_client_secret, datatable, teams_team_guid, auto_invite, error_handler, success_handler, public_app_execution_limit_per_minute, error_handler_fallback_to_instance_alerts, guest_access_enabled, guest_jwt_public_key, guest_jwt_jwks_url, add_admins_and_developers_to_forks, remote_deploy_target, remote_deploy_target_changed_at FROM workspace_settings WHERE workspace_id = $2", &rw.new_id, &old_id ) @@ -382,6 +382,13 @@ pub(crate) async fn change_workspace_id( ) .execute(&mut *tx) .await?; + sqlx::query!( + "UPDATE draft_move SET workspace_id = $1 WHERE workspace_id = $2", + &rw.new_id, + &old_id + ) + .execute(&mut *tx) + .await?; info!("Updating favorite table"); sqlx::query!( @@ -495,21 +502,25 @@ pub(crate) async fn change_workspace_id( // A fork's data table entry names the workspace that governs it by id, so the rename has to // follow there too — anywhere, not just in the reparented children: a detached workspace can // point at this one without being its fork. Left behind, the pointer resolves to the archived - // shell and every job through it stops. + // shell and every job through it stops. A clone's `governed_by` names it the same way. info!("Re-pointing data table references to the new workspace id"); sqlx::query!( r#"UPDATE workspace_settings ws SET datatable = ( SELECT jsonb_set(ws.datatable, '{datatables}', jsonb_object_agg( dt.key, - CASE WHEN dt.value->'reference'->>'workspace_id' = $2 - THEN jsonb_set(dt.value, '{reference,workspace_id}', to_jsonb($1::text)) - ELSE dt.value END + (SELECT CASE WHEN r.v->'governed_by'->>'workspace_id' = $2 + THEN jsonb_set(r.v, '{governed_by,workspace_id}', to_jsonb($1::text)) + ELSE r.v END + FROM (SELECT CASE WHEN dt.value->'reference'->>'workspace_id' = $2 + THEN jsonb_set(dt.value, '{reference,workspace_id}', to_jsonb($1::text)) + ELSE dt.value END AS v) r) )) FROM jsonb_each(ws.datatable->'datatables') dt ) WHERE jsonb_typeof(ws.datatable->'datatables') = 'object' - AND ws.datatable::text LIKE '%"reference"%'"#, + AND (ws.datatable::text LIKE '%"reference"%' + OR ws.datatable::text LIKE '%"governed_by"%')"#, &rw.new_id, &old_id, ) @@ -830,6 +841,14 @@ pub(crate) async fn change_workspace_id( .execute(&mut *tx) .await?; + sqlx::query!( + "UPDATE remote_deploy_token SET workspace_id = $1 WHERE workspace_id = $2", + &rw.new_id, + &old_id + ) + .execute(&mut *tx) + .await?; + info!("Updating variable table"); sqlx::query!( "UPDATE variable SET workspace_id = $1 WHERE workspace_id = $2", @@ -996,17 +1015,19 @@ pub(crate) async fn delete_workspace( // fails mid-way must never leave a live workspace with its fork data destroyed and no // registry row to retry from. Read-only: nothing is dropped here. // Read before the delete: another workspace's data table entry can point at one of this - // workspace's, and deleting the workspace it names leaves that pointer resolving to nothing. - // Nothing sweeps them — turning them back into copies would hand each fork the database - // outright — so the deleter is told which data tables they just stranded. - let stranded_pointers = sqlx::query!( - r#"SELECT ws.workspace_id AS "workspace_id!", dt.key AS "datatable!" + // workspace's, or be a clone taking its roles from one, and deleting the workspace it names + // leaves it resolving to nothing. Nothing sweeps them — turning them back into copies would + // hand each fork the database outright — so the deleter is told which data tables they just + // stranded. + let stranded_pointers = sqlx::query_as::<_, (String, String)>( + r#"SELECT ws.workspace_id, dt.key FROM workspace_settings ws CROSS JOIN LATERAL jsonb_each(COALESCE(ws.datatable->'datatables', '{}'::jsonb)) dt WHERE dt.value->'reference'->>'workspace_id' = $1 + OR dt.value->'governed_by'->>'workspace_id' = $1 ORDER BY ws.workspace_id, dt.key"#, - &w_id, ) + .bind(&w_id) .fetch_all(&db) .await .unwrap_or_default(); @@ -1334,7 +1355,7 @@ pub(crate) async fn delete_workspace( } else { let stranded = stranded_pointers .iter() - .map(|r| format!("{}/{}", r.workspace_id, r.datatable)) + .map(|(workspace_id, datatable)| format!("{workspace_id}/{datatable}")) .collect::>() .join(", "); Ok(format!( diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index d37c647db2..0dd9f8620e 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -1,7 +1,7 @@ openapi: "3.0.3" info: - version: 1.815.0 + version: 1.817.0 title: Windmill API contact: @@ -529,30 +529,6 @@ paths: responses: "302": description: redirected to the post-login destination, or to /user/login_link_expired when the link is used, expired or unknown - post: - security: [] - summary: consume a single-use login link from its confirmation page, set the session cookie and answer where to go - operationId: confirmLoginLink - tags: - - user - parameters: - - name: token - in: path - required: true - schema: - type: string - responses: - "200": - description: the post-login destination, or /user/login_link_expired when the link is used, expired or unknown - content: - application/json: - schema: - type: object - required: - - location - properties: - location: - type: string /auth/reset_password: post: @@ -4211,6 +4187,100 @@ paths: deploy_to: type: string + /w/{workspace}/remote_deploy/target: + get: + summary: get the workspace's deploy target on another instance, and the caller's connection to it + description: | + Once connected, deploy calls aimed at the target go through + `/w/{workspace}/remote_deploy/proxy/{proxy_key}/{route}`, which forwards any method to + `{base_url}/api/w/{target workspace}/{route}` with the caller's stored token. + operationId: getRemoteDeployTarget + tags: + - workspace + parameters: + - $ref: "#/components/parameters/WorkspaceId" + responses: + "200": + description: remote deploy target and the caller's connection to it + content: + application/json: + schema: + $ref: "#/components/schemas/RemoteDeployStatus" + post: + summary: set or clear the workspace's deploy target on another instance + description: Changing the target drops every token stored for the previous one. + operationId: setRemoteDeployTarget + tags: + - workspace + parameters: + - $ref: "#/components/parameters/WorkspaceId" + requestBody: + required: true + content: + application/json: + schema: + type: object + properties: + target: + $ref: "#/components/schemas/RemoteDeployTarget" + responses: + "200": + description: status + content: + text/plain: + schema: + type: string + + /w/{workspace}/remote_deploy/connect: + post: + summary: store the caller's own token for the remote deploy target + description: | + The token is checked against the target's whoami before it is stored. `target` names the + target the token was obtained for, and the request is refused if the workspace now points + elsewhere, so a token is never sent to an instance it was not meant for. + operationId: connectRemoteDeploy + tags: + - workspace + parameters: + - $ref: "#/components/parameters/WorkspaceId" + requestBody: + required: true + content: + application/json: + schema: + type: object + properties: + token: + type: string + target: + $ref: "#/components/schemas/RemoteDeployTarget" + required: + - token + - target + responses: + "200": + description: the stored connection + content: + application/json: + schema: + $ref: "#/components/schemas/RemoteDeployConnection" + + /w/{workspace}/remote_deploy/disconnect: + post: + summary: forget the caller's token for the remote deploy target + operationId: disconnectRemoteDeploy + tags: + - workspace + parameters: + - $ref: "#/components/parameters/WorkspaceId" + responses: + "200": + description: status + content: + text/plain: + schema: + type: string + /w/{workspace}/workspaces/is_premium: get: summary: get if workspace is premium @@ -5325,6 +5395,97 @@ paths: schema: type: string + /w/{workspace}/workspaces/datatable_acl/{datatable_name}: + get: + summary: read the owner and grants of an instance data table's database, schema or table + operationId: getDatatableAcl + tags: + - workspace + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: datatable_name + in: path + required: true + schema: + type: string + - name: kind + in: query + required: true + schema: + type: string + enum: [database, schema, table] + - name: schema + in: query + required: false + schema: + type: string + - name: table + in: query + required: false + schema: + type: string + responses: + "200": + description: owner and grants + content: + application/json: + schema: + $ref: "#/components/schemas/DatatableAclInfo" + + /w/{workspace}/workspaces/datatable_acl/{datatable_name}/plan: + post: + summary: preview the SQL an ownership or grant change would run (data table administrators only) + operationId: planDatatableAcl + tags: + - workspace + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: datatable_name + in: path + required: true + schema: + type: string + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/AclChangeRequest" + responses: + "200": + description: statements that would run, in a single transaction + content: + application/json: + schema: + $ref: "#/components/schemas/AclPlan" + + /w/{workspace}/workspaces/datatable_acl/{datatable_name}/apply: + post: + summary: run an ownership or grant change exactly as planned (data table administrators only) + operationId: applyDatatableAcl + tags: + - workspace + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: datatable_name + in: path + required: true + schema: + type: string + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/AclChangeRequest" + responses: + "200": + description: change applied + content: + text/plain: + schema: + type: string + /w/{workspace}/workspaces/datatable_usable_roles/{datatable_name}: get: summary: list the data table roles the caller may connect as @@ -5428,6 +5589,21 @@ paths: - workspace parameters: - $ref: "#/components/parameters/WorkspaceId" + - name: datatable_name + in: query + description: list only this data table; each listed data table opens a connection to its database + schema: + type: string + - name: role_for + in: query + description: the data table `role` applies to; every other one is listed as its default role + schema: + type: string + - name: role + in: query + description: the role to list `role_for` as; refused, in that entry's `error`, if the caller may not use it + schema: + type: string responses: "200": description: table metadata of all datatables @@ -5461,6 +5637,11 @@ paths: required: true schema: type: string + - name: role + in: query + description: the data table role to read the table as; defaults to the data table's default role + schema: + type: string responses: "200": description: schema of one datatable table @@ -6817,7 +6998,7 @@ paths: type: string expires_in_s: type: integer - description: link lifetime in seconds, at most 7200 (default 600) + description: link lifetime in seconds, at most 900 (default 600) rd: type: string description: same-origin path the browser lands on after login (default /user/workspaces) @@ -6827,12 +7008,6 @@ paths: mint only while the account still has this login type (for example pending_oauth), so a link stops working once the owner has set a password or signed in with a provider - confirm: - type: boolean - description: >- - return a /user/login_link page that signs in only when its button is - clicked, instead of a link spent by opening it; set it for links sent by - email, which mail scanners open on delivery (default false) responses: "201": description: login link minted @@ -10392,8 +10567,49 @@ paths: current_timestamp: type: string format: date-time + path: + type: string + description: "`saved` only, upsert or delete: where the write landed. Differs from the URL path when the item had moved away from it; the editor follows it there. Absent when a delete found nothing to remove and the caller cannot read the path it moved to." required: [status, current_timestamp] + /w/{workspace}/drafts/move/{kind}/{path}: + post: + summary: move the current user's draft-only item to another path + description: Relocates the authed user's own draft row, along with both path keys inside its value (the typed path and the mirror the editors keep beside it). Only for draft-only items — a deployed item must be moved through its own deploy endpoint, which carries every draft with it. Restricted to script, flow, app and raw_app; any other kind is rejected with 400, because only these keep their deploy target where this endpoint rewrites it. + operationId: moveDraft + tags: + - draft + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: kind + in: path + required: true + description: script, flow, app or raw_app only. + schema: + type: string + enum: [script, flow, app, raw_app] + - $ref: "#/components/parameters/ScriptPath" + requestBody: + required: true + content: + application/json: + schema: + type: object + properties: + new_path: + type: string + summary: + type: string + description: Also restate the draft's summary. + required: [new_path] + responses: + "200": + description: move result + content: + text/plain: + schema: + type: string + /w/{workspace}/drafts/migrate_legacy/{kind}/{path}: post: summary: resolve a legacy (workspace-level) draft (admin only) @@ -10904,6 +11120,8 @@ paths: parameters: - $ref: "#/components/parameters/WorkspaceId" - $ref: "#/components/parameters/ScriptPath" + - $ref: "#/components/parameters/Page" + - $ref: "#/components/parameters/PerPage" responses: "200": description: script history @@ -12309,6 +12527,8 @@ paths: parameters: - $ref: "#/components/parameters/WorkspaceId" - $ref: "#/components/parameters/ScriptPath" + - $ref: "#/components/parameters/Page" + - $ref: "#/components/parameters/PerPage" tags: - flow responses: @@ -14187,6 +14407,8 @@ paths: parameters: - $ref: "#/components/parameters/WorkspaceId" - $ref: "#/components/parameters/ScriptPath" + - $ref: "#/components/parameters/Page" + - $ref: "#/components/parameters/PerPage" responses: "200": description: app history @@ -14454,9 +14676,9 @@ paths: "200": description: app updated content: - text/plain: + application/json: schema: - type: string + $ref: "#/components/schemas/AppDeployed" /w/{workspace}/apps/create_raw_source: post: @@ -14596,9 +14818,9 @@ paths: "200": description: app updated content: - text/plain: + application/json: schema: - type: string + $ref: "#/components/schemas/AppDeployed" /w/{workspace}/apps/update_raw/{path}: post: @@ -14652,9 +14874,9 @@ paths: "200": description: app updated content: - text/plain: + application/json: schema: - type: string + $ref: "#/components/schemas/AppDeployed" /w/{workspace}/apps/custom_path_exists/{custom_path}: get: @@ -27702,6 +27924,13 @@ components: draft_saved_at: type: string format: date-time + draft_base: + type: string + description: | + The deployed version the draft forked from, as text whatever the + kind (script hash, flow version id, app version id). Compare to the + deployed head to tell a draft that is behind. Absent when there is + no draft or it was never forked from a deploy. no_deployed: type: boolean draft: @@ -28831,6 +29060,15 @@ components: type: object additionalProperties: $ref: "#/components/schemas/ModelPriceOverride" + context_window_per_model: + type: object + description: >- + Context window in tokens per `provider:model`, overriding the built-in + table the AI chat uses to decide when to compact its history. + additionalProperties: + type: integer + minimum: 1024 + maximum: 10000000 copilot_disabled: type: boolean description: >- @@ -29597,6 +29835,11 @@ components: type: string deployment_msg: type: string + created_at: + type: string + format: date-time + created_by: + type: string required: - script_hash @@ -33652,6 +33895,8 @@ components: $ref: "#/components/schemas/LoggedWizardStatus" replication_user_error: type: string + user_connect: + $ref: "#/components/schemas/LoggedWizardStatus" CustomInstanceDbTag: type: string @@ -33702,6 +33947,15 @@ components: $ref: "#/components/schemas/DatatableRoleTenants" governing_workspace_id: type: string + clone_of: + type: object + description: for a clone, the data table whose roles it takes + required: [workspace_id, datatable] + properties: + workspace_id: + type: string + datatable: + type: string editable: type: boolean available_roles: @@ -33719,6 +33973,258 @@ components: datatable: type: string + AclTarget: + description: what access is read or changed on + oneOf: + - $ref: "#/components/schemas/AclTargetDatabase" + - $ref: "#/components/schemas/AclTargetSchema" + - $ref: "#/components/schemas/AclTargetTable" + discriminator: + propertyName: kind + mapping: + database: "#/components/schemas/AclTargetDatabase" + schema: "#/components/schemas/AclTargetSchema" + table: "#/components/schemas/AclTargetTable" + + AclTargetDatabase: + type: object + required: [kind] + properties: + kind: + type: string + enum: [database] + + AclTargetSchema: + type: object + required: [kind, schema] + properties: + kind: + type: string + enum: [schema] + schema: + type: string + + AclTargetTable: + type: object + required: [kind, schema, table] + properties: + kind: + type: string + enum: [table] + schema: + type: string + table: + type: string + + AclChange: + description: one change to plan or apply + oneOf: + - $ref: "#/components/schemas/AclChangeSetOwner" + - $ref: "#/components/schemas/AclChangeGrant" + - $ref: "#/components/schemas/AclChangeRevoke" + discriminator: + propertyName: type + mapping: + set_owner: "#/components/schemas/AclChangeSetOwner" + grant: "#/components/schemas/AclChangeGrant" + revoke: "#/components/schemas/AclChangeRevoke" + + AclChangeSetOwner: + type: object + description: >- + hands the target to role — for a schema, with everything already in it but an extension's + members, which stay with the extension + required: [type, role] + properties: + type: + type: string + enum: [set_owner] + role: + type: string + description: a data table role of the instance, or admin + + AclChangeGrant: + type: object + required: [type, role, privileges, scope] + properties: + type: + type: string + enum: [grant] + role: + type: string + description: a data table role of the instance, or admin + privileges: + type: array + items: + type: string + scope: + $ref: "#/components/schemas/AclGrantScope" + + AclChangeRevoke: + type: object + required: [type, role, privileges, scope] + properties: + type: + type: string + enum: [revoke] + role: + type: string + description: a data table role of the instance, other than admin + privileges: + type: array + items: + type: string + scope: + $ref: "#/components/schemas/AclGrantScope" + objects: + type: array + description: >- + objects inside the target the revoke covers, empty for the target itself. Only with the + target scope; a revoke on all objects of a kind is refused, since it cannot say which + grants it takes back. + items: + $ref: "#/components/schemas/AclObject" + + AclGrantScope: + type: string + enum: + [ + target, + all_tables, + all_sequences, + all_functions, + future_tables, + future_sequences, + future_functions, + ] + + AclChangeRequest: + type: object + required: [target, change] + properties: + target: + $ref: "#/components/schemas/AclTarget" + change: + $ref: "#/components/schemas/AclChange" + statements: + type: array + description: >- + The statements the plan showed. Required to apply, which plans again and refuses if + the result differs. + items: + type: string + + AclPlan: + type: object + required: [statements, warnings] + properties: + statements: + type: array + items: + type: string + warnings: + type: array + items: + type: string + + AclObject: + type: object + required: [name, kind] + properties: + name: + type: string + kind: + type: string + description: >- + TABLE, SEQUENCE, FUNCTION, PROCEDURE or TYPE — what the object is. A revoke turns it + into the keyword it takes, ROUTINE for both routine kinds; a type's grants are read + only. + args: + type: string + description: identity arguments of a routine, which is what tells two of the same name apart + + AclGrant: + type: object + required: [grantee, privileges, sources] + properties: + grantee: + type: string + privileges: + type: array + items: + type: string + object: + $ref: "#/components/schemas/AclObject" + future: + type: string + description: >- + set for a default privilege, naming the kind of object it covers (TABLES, SEQUENCES, + FUNCTIONS, TYPES, or SCHEMAS). On a schema, the defaults set in that schema; on the + database, the ones set database-wide, which apply in every schema and which no schema's + own defaults take back. + sources: + type: array + description: >- + the roles the grant comes from, each once — who granted it, or for a default privilege + the role whose future objects it covers. A revoke of some of the grant's privileges + takes them back from every source that gave them. + items: + $ref: "#/components/schemas/AclSource" + + AclSource: + type: object + required: [role, privileges, reachable] + properties: + role: + type: string + privileges: + type: array + description: >- + what role gave of the grant's privileges. A revoke is held back only by a source out of + reach that gave some of what it takes back. + items: + type: string + reachable: + type: boolean + description: >- + whether the data table's connection can take back what role gave. On an object that is + the owner, when the connection acts for the owner, and otherwise the connection itself; + for a default privilege, a creating role the connection acts for. What a source out of + reach gave is not revocable from here; privileges only other sources gave still are. + + DatatableAclInfo: + type: object + required: [owner, roles, editable, clone, supports_maintain, dbname, grants, children] + properties: + owner: + type: string + roles: + type: array + description: the roles a change may name; empty unless the caller may change anything + items: + type: string + editable: + type: boolean + description: whether the caller may plan and apply changes + clone: + type: boolean + description: whether this is a clone, whose grants stay as they were copied + supports_maintain: + type: boolean + description: whether the server is Postgres 17+, which added the MAINTAIN table privilege + dbname: + type: string + description: the database the target lives in + grants: + type: array + items: + $ref: "#/components/schemas/AclGrant" + children: + type: array + description: a database's schemas, or a schema's tables + items: + type: string + CustomInstanceDb: type: object required: @@ -34855,6 +35361,16 @@ components: new_dbname: type: string description: "New database name for the fork" + fork_behavior: + type: string + enum: + - schema_only + - schema_and_data + description: >- + What the fork request copies into `new_dbname`, which it creates — with the + owners and grants of a data table under roles. This server refuses an entry + without it; servers predating it expect `new_dbname` created and filled + beforehand. shared_ducklakes: type: array items: @@ -35493,6 +36009,20 @@ components: - raw_app + AppDeployed: + type: object + description: "What a deploy of an existing app answers with. `version` is the one this call wrote, which is what an editor pins as the fork base of the draft it starts next: reading the head back afterwards cannot tell it from a deploy that landed beside it. A metadata-only update writes none and reports the head it kept." + properties: + path: + type: string + description: Where the app now lives, which differs from the request path on a rename. + version: + type: integer + format: int64 + required: + - path + - version + AppHistory: type: object properties: @@ -35500,6 +36030,11 @@ components: type: integer deployment_msg: type: string + created_at: + type: string + format: date-time + created_by: + type: string required: - version @@ -35569,6 +36104,8 @@ components: format: date-time deployment_msg: type: string + created_by: + type: string required: - id - created_at @@ -35797,6 +36334,17 @@ components: type: string datatable: type: string + governed_by: + description: >- + On a clone, the data table it was copied from, whose roles it takes. Server-owned + like `reference`. + type: object + required: [workspace_id, datatable] + properties: + workspace_id: + type: string + datatable: + type: string migrations_enabled: type: boolean description: Whether the SQL migrations feature is opted in for this data table @@ -35865,7 +36413,17 @@ components: DataTableTables: type: object - required: [datatable_name, schemas] + required: + [ + datatable_name, + schemas, + instance, + permissioned, + usable_roles, + default_role, + can_create_schema, + creatable_schemas, + ] properties: datatable_name: type: string @@ -35878,6 +36436,26 @@ components: type: string error: type: string + instance: + type: boolean + description: on the instance database, the only kind that can be under roles or have its access edited + permissioned: + type: boolean + usable_roles: + type: array + description: the roles the caller may connect as, by name; empty when not under roles + items: + type: string + default_role: + type: string + can_create_schema: + type: boolean + description: whether the role the listing connected as may create schemas + creatable_schemas: + type: array + description: the schemas the role the listing connected as may create in + items: + type: string DataTableTableSchema: type: object @@ -36048,6 +36626,44 @@ components: items: $ref: "#/components/schemas/GitSyncObjectType" + RemoteDeployTarget: + type: object + properties: + base_url: + type: string + description: root URL of the remote Windmill instance, without /api + workspace_id: + type: string + description: workspace on the remote instance that deploys land in + required: + - base_url + - workspace_id + + RemoteDeployConnection: + type: object + properties: + remote_email: + type: string + description: identity the stored token has on the remote instance + proxy_key: + type: string + description: goes in every proxy URL, `/w/{workspace}/remote_deploy/proxy/{proxy_key}/{route}` + connected_at: + type: string + format: date-time + required: + - remote_email + - proxy_key + - connected_at + + RemoteDeployStatus: + type: object + properties: + target: + $ref: "#/components/schemas/RemoteDeployTarget" + connection: + $ref: "#/components/schemas/RemoteDeployConnection" + WorkspaceDefaultScripts: type: object properties: diff --git a/backend/windmill-api/src/ai.rs b/backend/windmill-api/src/ai.rs index 1710db9cec..16b4b17141 100644 --- a/backend/windmill-api/src/ai.rs +++ b/backend/windmill-api/src/ai.rs @@ -23,7 +23,7 @@ use windmill_ai::ai_cache::current_instance_ai_config_revision; use windmill_ai::ai_providers::{ empty_string_as_none, AIPlatform, AIProvider, ProviderConfig, ProviderModel, }; -use windmill_ai::ai_types::MAX_MODEL_RATE; +use windmill_ai::ai_types::{validate_token_limit, CONTEXT_WINDOWS, MAX_MODEL_RATE, OUTPUT_LIMITS}; use windmill_ai::credentials::ProviderCredentials; #[cfg(feature = "bedrock")] use windmill_ai::providers::bedrock::{ @@ -445,6 +445,11 @@ pub struct AIConfig { /// Only models whose rates differ from the built-in table are stored. #[serde(skip_serializing_if = "Option::is_none")] pub model_pricing: Option>, + /// Per-model context windows the chat budgets its history against, keyed + /// `provider:model` like `max_tokens_per_model`. Overrides the client's built-in + /// table, which a model served through a custom endpoint is usually missing from. + #[serde(skip_serializing_if = "Option::is_none")] + pub context_window_per_model: Option>, /// Hides the Windmill AI assistant (chat, sessions, generation, completion, fixes) from /// the workspace UI. Only the workspace's own row is consulted: the flag holds even when /// the providers served come from the instance config or the free tier. AI agent steps @@ -508,6 +513,18 @@ impl AIConfig { Ok(()) } + pub fn validate_token_limits(&self) -> Result<()> { + for (entries, bounds) in [ + (&self.context_window_per_model, &CONTEXT_WINDOWS), + (&self.max_tokens_per_model, &OUTPUT_LIMITS), + ] { + for (key, tokens) in entries.iter().flatten() { + validate_token_limit(bounds, key, i64::from(*tokens)).map_err(Error::BadRequest)?; + } + } + Ok(()) + } + pub fn validate_sessions_retention(&self) -> Result<()> { match self.sessions_retention_days { Some(days) if !(1..=MAX_SESSIONS_RETENTION_DAYS).contains(&days) => { diff --git a/backend/windmill-api/src/apps.rs b/backend/windmill-api/src/apps.rs index 918fce53bf..edd2040143 100644 --- a/backend/windmill-api/src/apps.rs +++ b/backend/windmill-api/src/apps.rs @@ -69,8 +69,8 @@ use windmill_common::{ user_drafts::{overlay_or_draft_only, DraftUserRef, UserDraftItemKind, WithDraftOverlay}, users::username_to_permissioned_as, utils::{ - http_get_from_hub, not_found_if_none, paginate, query_elems_from_hub, require_admin, - strip_json_nul, Pagination, RunnableKind, StripPath, + http_get_from_hub, not_found_if_none, paginate, paginate_optional, + query_elems_from_hub, require_admin, strip_json_nul, Pagination, RunnableKind, StripPath, }, variables::{build_crypt, build_crypt_with_key_suffix, encrypt}, worker::{to_raw_value, CLOUD_HOSTED}, @@ -274,6 +274,12 @@ pub struct AppHistory { pub version: i64, #[serde(skip_serializing_if = "Option::is_none")] pub deployment_msg: Option, + /// Who deployed this version, and when — the diff's version picker names them so + /// a reader can tell their own deploys from a teammate's. + #[serde(skip_serializing_if = "Option::is_none")] + pub created_by: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub created_at: Option>, } #[derive(Deserialize)] @@ -567,6 +573,17 @@ pub struct CreateApp { pub skip_draft_deletion: Option, } +/// What a deploy of an existing app answers with. `version` is the one this call wrote, +/// which is what an editor pins as the fork base of the draft it starts next: reading the +/// head back afterwards cannot tell it from a deploy that landed beside it. A +/// metadata-only update writes none and reports the head it kept. +#[derive(Serialize)] +pub struct AppDeployed { + /// Where the app now lives, which differs from the request path on a rename. + pub path: String, + pub version: i64, +} + #[derive(Serialize, Deserialize)] pub struct EditApp { pub path: Option, @@ -1220,18 +1237,42 @@ async fn get_app_history( authed: ApiAuthed, Extension(user_db): Extension, Path((w_id, path)): Path<(String, StripPath)>, + Query(pagination): Query, ) -> JsonResult> { let path = path.to_path(); check_scopes(&authed, || format!("apps:read:{}", &path))?; + // Unasked-for, this listing stays whole: the deployment-history panel reads it + // without paging. The diff picker asks for a page. + let (per_page, offset) = paginate_optional(pagination); let mut tx = user_db.begin(&authed).await?; + // Newest first in deployed order, which is a version's position in `app.versions` and + // not its `created_at`: the latter is the deploying transaction's start time, so two + // that overlap can carry it in the opposite order from the one they landed in. A row + // outside the array never sat in that sequence, so it sorts after the ones that did. + // Paging happens before the metadata joins, so a page costs its own rows. let query_result = sqlx::query!( - "SELECT a.id as app_id, av.id as version_id, dm.deployment_msg as deployment_msg - FROM app a LEFT JOIN app_version av ON a.id = av.app_id LEFT JOIN deployment_metadata dm ON av.id = dm.app_version + "SELECT a.id as app_id, av.id as version_id, dm.deployment_msg as deployment_msg, + av.created_by as created_by, av.created_at as created_at + FROM app a + JOIN LATERAL ( + SELECT av2.id, COALESCE(v.ord, 0) AS ord + FROM app_version av2 + LEFT JOIN unnest(a.versions) WITH ORDINALITY AS v(id, ord) ON v.id = av2.id + WHERE av2.app_id = a.id + ORDER BY ord DESC, av2.id DESC + LIMIT $3 OFFSET $4 + ) page ON TRUE + JOIN app_version av ON av.id = page.id + LEFT JOIN deployment_metadata dm ON av.id = dm.app_version WHERE a.workspace_id = $1 AND a.path = $2 - ORDER BY created_at DESC", + ORDER BY page.ord DESC", w_id, path, - ).fetch_all(&mut *tx).await?; + per_page, + offset, + ) + .fetch_all(&mut *tx) + .await?; tx.commit().await?; let result: Vec = query_result @@ -1240,6 +1281,8 @@ async fn get_app_history( app_id: row.app_id, version: row.version_id, deployment_msg: row.deployment_msg, + created_by: Some(row.created_by), + created_at: Some(row.created_at), }) .collect(); return Ok(Json(result)); @@ -1253,14 +1296,22 @@ async fn get_latest_version( let path = path.to_path(); check_scopes(&authed, || format!("apps:read:{}", path))?; let mut tx = user_db.begin(&authed).await?; + // The head is the tail of `app.versions` — the version the runtime serves. Deploys + // append to it under the app row's lock, whereas `app_version.created_at` is the + // deploying transaction's start time, so two that overlap can carry it in either + // order and the newest timestamp is then not the one that landed last. let row = sqlx::query!( - "SELECT a.id as app_id, av.id as version_id, dm.deployment_msg as deployment_msg - FROM app a LEFT JOIN app_version av ON a.id = av.app_id LEFT JOIN deployment_metadata dm ON av.id = dm.app_version - WHERE a.workspace_id = $1 AND a.path = $2 - ORDER BY created_at DESC", + "SELECT a.id as app_id, av.id as version_id, dm.deployment_msg as deployment_msg, + av.created_by as created_by, av.created_at as created_at + FROM app a JOIN app_version av + ON av.id = a.versions[array_upper(a.versions, 1)] AND av.app_id = a.id + LEFT JOIN deployment_metadata dm ON av.id = dm.app_version + WHERE a.workspace_id = $1 AND a.path = $2", w_id, path, - ).fetch_optional(&mut *tx).await?; + ) + .fetch_optional(&mut *tx) + .await?; tx.commit().await?; if let Some(row) = row { @@ -1268,6 +1319,8 @@ async fn get_latest_version( app_id: row.app_id, version: row.version_id, deployment_msg: row.deployment_msg, + created_by: Some(row.created_by), + created_at: Some(row.created_at), }; return Ok(Json(Some(result))); @@ -2591,6 +2644,14 @@ async fn create_app_internal<'a>( .execute(&mut *tx) .await?; } + windmill_common::user_drafts::clear_draft_moves_from( + &mut tx, + &w_id, + &[UserDraftItemKind::App, UserDraftItemKind::RawApp], + &app.path, + None, + ) + .await?; let id = sqlx::query_scalar!( "INSERT INTO app (workspace_id, path, summary, policy, versions, custom_path, labels) @@ -2947,7 +3008,7 @@ async fn update_app( Extension(webhook): Extension, Path((w_id, path)): Path<(String, StripPath)>, Json(ns): Json, -) -> Result { +) -> JsonResult { if authed.is_operator { return Err(Error::NotAuthorized( "Operators cannot update apps for security reasons".to_string(), @@ -2986,7 +3047,7 @@ async fn update_app( }, ); - Ok(format!("app {} updated (npath: {:?})", opath, npath)) + Ok(Json(AppDeployed { path: npath, version: v_id })) } /// Deploy a raw app from its sources, compiling them on a worker. `update_raw` @@ -3000,7 +3061,7 @@ async fn update_app_raw_source( Extension(webhook): Extension, Path((w_id, path)): Path<(String, StripPath)>, Json(ns): Json, -) -> Result { +) -> JsonResult { if authed.is_operator { return Err(Error::NotAuthorized( "Operators cannot update apps for security reasons".to_string(), @@ -3098,7 +3159,7 @@ async fn update_app_raw_source( }, ); - Ok(format!("app {} updated (npath: {:?})", opath, npath)) + Ok(Json(AppDeployed { path: npath, version: v_id })) } /// Whether the caller may create an app at `path` — asked of the database rather @@ -3324,7 +3385,7 @@ async fn update_app_raw<'a>( Extension(webhook): Extension, Path((w_id, path)): Path<(String, StripPath)>, multipart: Multipart, -) -> Result { +) -> JsonResult { if authed.is_operator { return Err(Error::NotAuthorized( "Operators cannot update apps for security reasons".to_string(), @@ -3372,7 +3433,7 @@ async fn update_app_raw<'a>( }, ); - Ok(format!("app {} updated (npath: {:?})", opath, npath)) + Ok(Json(AppDeployed { path: npath, version: v_id })) } // async fn create_app_internal<'a>( // authed: ApiAuthed, @@ -3749,6 +3810,19 @@ async fn update_app_internal<'a>( .execute(&mut *tx) .await?; } + if npath != path { + // Everything left at the old path is a draft this deploy didn't consume + // — teammates' rows, and the deployer's own when the caller asked us to + // keep it. Carry them rather than strand them. + windmill_common::user_drafts::move_drafts_for_path( + &mut tx, + &w_id, + &[UserDraftItemKind::App, UserDraftItemKind::RawApp], + path, + &npath, + ) + .await?; + } audit_log( &mut *tx, &authed, @@ -4376,9 +4450,11 @@ async fn execute_component( // — like `/jobs/run/preview` — confine it to worker tags the caller may use // (a `if_jobs:filter_tags`-restricted token must not escape its filter). // `is_preview` implies an authed caller (the guard above returns otherwise). + let push_args = PushArgs { args: &args.args, extra: args.extra }; if is_preview { if let Some(authed) = opt_authed.as_ref() { - crate::jobs::check_tag_available_for_workspace(&db, &w_id, &tag, authed).await?; + crate::jobs::check_tag_available_for_workspace(&db, &w_id, &tag, &push_args, authed) + .await?; } } // Identity is already resolved to the requesting user in preview mode (the @@ -4409,7 +4485,7 @@ async fn execute_component( tx, &w_id, job_payload, - PushArgs { args: &args.args, extra: args.extra }, + push_args, &username, email, permissioned_as, diff --git a/backend/windmill-api/src/db.rs b/backend/windmill-api/src/db.rs index e8a6858db7..9619e96ebc 100644 --- a/backend/windmill-api/src/db.rs +++ b/backend/windmill-api/src/db.rs @@ -372,7 +372,10 @@ pub async fn migrate( } crate::live_migrations::custom_migrations(&mut custom_migrator).await?; - Ok(None) + Ok(Some(crate::live_migrations::spawn_background_migrations( + db.clone(), + killpill_rx, + ))) } pub async fn wait_for_migrations( diff --git a/backend/windmill-api/src/drafts.rs b/backend/windmill-api/src/drafts.rs index e5928f761e..8bcbb56384 100644 --- a/backend/windmill-api/src/drafts.rs +++ b/backend/windmill-api/src/drafts.rs @@ -19,7 +19,7 @@ use windmill_common::{ error::{Error, Result}, user_drafts::{DraftUserRef, UserDraftItemKind, ENCRYPTED_DRAFT_PREFIX}, users::resolve_username_to_email, - utils::strip_json_nul, + utils::{check_proper_path, strip_json_nul}, variables::{build_crypt, encrypt}, }; @@ -29,6 +29,7 @@ pub fn workspaced_service() -> Router { .route("/get/{kind}/{*path}", get(get_draft_for_user)) .route("/get_own/{kind}/{*path}", get(get_own_draft)) .route("/update/{kind}/{*path}", post(update_draft)) + .route("/move/{kind}/{*path}", post(move_draft)) .route("/migrate_legacy/{kind}/{*path}", post(migrate_legacy_draft)) } @@ -320,12 +321,71 @@ pub struct SaveDraftResponse { /// On `saved`: when the change was applied (client remembers it as the /// next `last_sync`). On `conflict`: the existing row's `created_at`. pub current_timestamp: chrono::DateTime, + /// `saved` only: where the write landed. Differs from the URL path when the item + /// had moved away from it; the editor follows it there. Absent when a delete found + /// nothing to remove and the caller cannot read the path it moved to. + #[serde(skip_serializing_if = "Option::is_none")] + pub path: Option, } -/// Apply the current user's draft at (workspace, kind, path): non-null `value` -/// upserts, `null` (or omitted) deletes. Either way, when the existing row is -/// newer than `last_sync` (and `force` is false) the op is skipped and the -/// response is `status = conflict` + the server's current timestamp. +/// The version a draft forked from, as the editors write it into `draft.value`. +/// Each kind names it differently and only one is ever set. +#[derive(Deserialize)] +struct DraftBaseVersion { + /// Scripts: hex-encoded script hash. + #[serde(default)] + parent_hash: Option, + /// Flows: `flow_version.id`. + #[serde(default)] + version_id: Option, + /// Apps / raw apps: `app_version.id`. + #[serde(default)] + parent_version: Option, +} + +impl DraftBaseVersion { + /// The base as the `draft.base` column stores it: one opaque text id whatever + /// the kind, so a reader compares it to the head without knowing the kind's + /// own field name or type. + fn as_text(&self, kind: UserDraftItemKind) -> Option { + use UserDraftItemKind::*; + match kind { + Script => self.parent_hash.clone(), + Flow => self.version_id.map(|v| v.to_string()), + _ => self.parent_version.map(|v| v.to_string()), + } + } +} + +/// The version this draft forked from, or `None` when it has none — a kind +/// that keeps no lineage, a malformed payload, or a draft that was never forked +/// from a deploy. Pure: no queries. +fn draft_lineage(kind: UserDraftItemKind, value: &str) -> Option { + use UserDraftItemKind::*; + if !matches!(kind, Script | Flow | App | RawApp) { + return None; + } + let base = serde_json::from_str::(value).ok()?; + let has_base = match kind { + Script => base + .parent_hash + .as_deref() + .and_then(|h| windmill_common::scripts::to_i64(h).ok()) + .is_some(), + Flow => base.version_id.is_some(), + _ => base.parent_version.is_some(), + }; + has_base.then_some(base) +} + +/// Apply the current user's draft: non-null `value` upserts, `null` (or +/// omitted) deletes. Either way, when the existing row is newer than +/// `last_sync` (and `force` is false) the op is skipped and the response is +/// `status = conflict` + the server's current timestamp. +/// +/// A save addressed to a path its item moved away from lands where the move took +/// the drafts (`draft_move`), unless the caller still has a draft of their own at +/// that path. The response names where it landed. async fn update_draft( authed: ApiAuthed, Extension(db): Extension, @@ -334,7 +394,7 @@ async fn update_draft( Json(req): Json, ) -> Result> { let email = &authed.email; - let path = path.to_path(); + let url_path = path.to_path(); // Saving a draft requires write permission on the underlying path. Deleting // (discarding) one's OWN draft does not: the email-scoped row belongs to the // authed user, so they can always discard it even after losing write access @@ -343,11 +403,68 @@ async fn update_draft( // touch the caller's own row. Legacy (NULL-email) rows aren't owned by anyone // — they keep the write gate. let is_own_discard = req.value.is_none() && !req.legacy; + // `legacy` targets the workspace-level row and is delete-only: an upsert writes the + // caller's own row whatever it says. Every read of that rule goes through this. + let legacy_delete = req.value.is_none() && req.legacy; + + // Whose row this write is for: the caller's, or the workspace-level one on a legacy + // DELETE (`legacy` is delete-only, so an upsert is the caller's own row either way). + // It picks both the record that applies — an item's move (`email IS NULL`) covers the + // legacy row too, since the same rename carried it — and the draft whose presence + // means this path is still the write's own. + let owner: Option<&str> = (!legacy_delete).then_some(email.as_str()); + // The caller's own draft-only move outranks the move of the deployed item. Only the + // kinds whose value carries a deploy target are ever recorded as moved, so for the + // rest this would be a guaranteed-empty query on the autosave hot path. + let moved_to = match kind.typed_path_field() { + None => None, + Some(_) => { + sqlx::query_scalar!( + r#"SELECT m.new_path FROM draft_move m + WHERE m.workspace_id = $1 AND m.typ = $2 AND m.old_path = $3 + AND (m.email IS NULL OR m.email = $4) + AND NOT EXISTS ( + SELECT 1 FROM draft d + WHERE d.workspace_id = $1 AND d.typ = $2 AND d.path = $3 + AND d.email IS NOT DISTINCT FROM $4 + ) + ORDER BY m.email IS NULL + LIMIT 1"#, + &w_id, + kind as UserDraftItemKind, + url_path, + owner, + ) + .fetch_optional(&db) + .await? + } + }; + let path: &str = moved_to.as_deref().unwrap_or(url_path); + + // Everything past here writes, so the gate applies from here on. Answered + // without the path when the item moved: it may have gone somewhere the caller + // cannot see. if !is_own_discard { - require_can_write_path(&authed, &db, &user_db, &w_id, kind, path).await?; + match require_can_write_path(&authed, &db, &user_db, &w_id, kind, path).await { + // Naming the move is for whoever was editing the item: it tells them why a + // save they were already making stopped landing. Someone who cannot read the + // path they addressed gets the plain denial, or the wording itself would + // answer whether an item was moved away from a path they only guessed at. + Err(Error::NotAuthorized(e)) if moved_to.is_some() => { + return Err(Error::NotAuthorized( + match require_can_read_path(&authed, &user_db, &w_id, kind, url_path).await { + Ok(()) => { + "this draft's item was moved to a path you cannot write".to_string() + } + Err(_) => e, + }, + )); + } + other => other?, + } } - let applied_at = if let Some(value) = &req.value { + let applied = if let Some(value) = &req.value { // Secret variable values must never sit in `draft.value` in plaintext // (see `encrypt_secret_variable_value`). let serialized = if kind == UserDraftItemKind::Variable { @@ -359,19 +476,49 @@ async fn update_draft( // escape and later make any `->>`/`to_jsonb` extraction raise `22P05`. // Strip it here so a NUL never reaches the column. let serialized = strip_json_nul(&serialized); + // `base` is derived here from the value's per-kind field rather than sent + // by the client, so every writer (editors, chat, CLI) fills it the same way. + // Read from the sanitized text, which is what the value column gets: a NUL in the + // lineage field otherwise costs the draft its base (the hash no longer parses), + // leaving a draft that reads as up to date whatever the head is. + let base = draft_lineage(kind, serialized.as_ref()).and_then(|l| l.as_text(kind)); // Upsert. The conflict check rides on the DO UPDATE WHERE clause — // when the row is newer than `last_sync`, RETURNING yields nothing. // `created_at` defaults to `now()` but the migration overrides it ($8) // so a migrated draft keeps its original age instead of jumping to top. - sqlx::query_scalar!( - r#"INSERT INTO draft (workspace_id, email, path, typ, value, created_at) - VALUES ($1, $2, $3, $4, $5::text::json, COALESCE($8::timestamptz, now())) + // + // A moved save ($10) carries the path keys its editor had before the move. + // One naming the path it addressed ($11) follows to where it landed; a draft + // already there keeps the keys the move gave it. A pre-sanitizer NUL escape + // in that draft makes `to_jsonb` raise, so it takes the incoming keys. + sqlx::query!( + r#"INSERT INTO draft (workspace_id, email, path, typ, value, created_at, base) + VALUES ($1, $2, $3::text, $4, + CASE WHEN $10::bool + THEN to_json($5::text::jsonb || jsonb_strip_nulls(jsonb_build_object( + 'path', CASE WHEN $5::text::jsonb -> 'path' = to_jsonb($11::text) + THEN to_jsonb($3::text) END, + 'draft_path', CASE WHEN $5::text::jsonb -> 'draft_path' = to_jsonb($11::text) + THEN to_jsonb($3::text) END))) + ELSE $5::text::json + END, + COALESCE($8::timestamptz, now()), $9) ON CONFLICT (workspace_id, path, typ, email) WHERE email IS NOT NULL - DO UPDATE SET value = EXCLUDED.value, created_at = EXCLUDED.created_at + DO UPDATE SET value = CASE + WHEN NOT $10::bool + OR position(chr(92) || 'u0000' in replace(draft.value::text, chr(92) || chr(92), '')) > 0 + THEN EXCLUDED.value + ELSE to_json((to_jsonb(EXCLUDED.value) - 'path' - 'draft_path') + || jsonb_strip_nulls(jsonb_build_object( + 'path', to_jsonb(draft.value) -> 'path', + 'draft_path', to_jsonb(draft.value) -> 'draft_path'))) + END, + created_at = EXCLUDED.created_at, + base = EXCLUDED.base WHERE $7::bool = true OR $6::timestamptz IS NULL OR draft.created_at <= $6::timestamptz - RETURNING created_at"#, + RETURNING path, created_at"#, &w_id, email, path, @@ -380,9 +527,13 @@ async fn update_draft( req.last_sync, req.force, req.created_at, + base.as_deref(), + moved_to.is_some(), + url_path, ) .fetch_optional(&db) .await? + .map(|r| (r.created_at, Some(r.path))) } else { // Delete, same conflict rule in the WHERE clause. Returns NULL when // the row was too new (conflict) OR already absent (idempotent) — @@ -403,16 +554,20 @@ async fn update_draft( kind as UserDraftItemKind, req.last_sync, req.force, - req.legacy, + legacy_delete, ) .fetch_optional(&db) .await? + // Named for the same reason an upsert is: the editor that discarded is still on + // the path the item left, and reloading there would land on nothing. + .map(|ts| (ts, moved_to.clone())) }; - if let Some(ts) = applied_at { + if let Some((ts, path)) = applied { return Ok(Json(SaveDraftResponse { status: SaveDraftStatus::Saved, current_timestamp: ts, + path, })); } @@ -428,7 +583,7 @@ async fn update_draft( email, path, kind as UserDraftItemKind, - req.legacy, + legacy_delete, ) .fetch_optional(&db) .await?; @@ -437,20 +592,307 @@ async fn update_draft( Some(ts) => Ok(Json(SaveDraftResponse { status: SaveDraftStatus::Conflict, current_timestamp: ts, + path: None, })), // Delete + nothing-was-there ⇒ report success with server's NOW(). None => { let now = sqlx::query_scalar!(r#"SELECT now() as "now!""#) .fetch_one(&db) .await?; + // A retry of a routed discard whose answer was lost lands here: the row is + // gone but the editor is still on the path the item left, so it needs the + // destination as much as the first attempt did. Unlike the arm above there is + // no deleted row proving the caller ever held that draft, and an own discard + // is not gated, so this names a path to someone who may have none of it: the + // read gate, which is what keeps a path from being disclosed elsewhere in this + // module. Without it, discarding at a guessed path reads `draft_move`. + let disclosed = match moved_to { + Some(dest) => { + match require_can_read_path(&authed, &user_db, &w_id, kind, &dest).await { + Ok(()) => Some(dest), + Err(Error::NotFound(_)) + | Err(Error::NotAuthorized(_)) + | Err(Error::BadRequest(_)) => None, + Err(e) => return Err(e), + } + } + None => None, + }; Ok(Json(SaveDraftResponse { status: SaveDraftStatus::Saved, current_timestamp: now, + path: disclosed, })) } } } +#[derive(Deserialize)] +pub struct MoveDraftRequest { + pub new_path: String, + /// Also restate the draft's summary, so the same drawer that renames a + /// deployed item can retitle a draft-only one. + #[serde(default)] + pub summary: Option, +} + +/// Relocate the authed user's own DRAFT-ONLY item. Such an item is nothing but +/// its draft row, so moving it is a rewrite of that row's path plus both path +/// keys inside its value — there is no deployed row, schedule or trigger to +/// cascade to. +/// +/// The owner's own open editor follows: its next save, still addressed to the old +/// path, lands at the new one through the move record, and it is told where. +/// +/// Scoped to the caller's own row on purpose: two users can each have a draft +/// at the same never-deployed path, and those are two separate items. +/// +/// A DEPLOYED item must move through its own deploy endpoint instead, which +/// cascades everything that references the path and carries every draft along. +async fn move_draft( + authed: ApiAuthed, + Extension(db): Extension, + Extension(user_db): Extension, + Path((w_id, kind, path)): Path<(String, UserDraftItemKind, windmill_common::utils::StripPath)>, + Json(req): Json, +) -> Result { + let path = path.to_path(); + let new_path = req.new_path.as_str(); + // Only the full-page editor kinds, which is exactly the set that has a typed + // path to rewrite. Reading the movable set off the same mapping the rewrite + // uses keeps them from drifting apart: a resource, a variable or a trigger + // keeps its deploy target in `value.path` with no editor to stage a rename, + // so moving one would leave the real target naming the old location and the + // next deploy would recreate it where it came from. + let (Some(typed_field), Some(mirror_field)) = + (kind.typed_path_field(), kind.mirror_path_field()) + else { + return Err(Error::BadRequest(format!( + "moving a {kind:?} draft is not supported — only scripts, flows and apps" + ))); + }; + // Validate before authorizing: `require_can_write_path` is not a format check + // (an admin returns immediately, and a user returns early inside their own + // namespace), so without this a malformed path is stored as-is, and an over-long + // or NUL-bearing one reaches Postgres as a raw server error. The source is a URL + // segment and arrives decoded, so it needs the same check as the destination. + check_proper_path(path)?; + check_proper_path(new_path)?; + // A summary-only edit is a legitimate use of this endpoint: the drawer edits + // both fields, and for a draft-only script the path it posts back is the row + // path unchanged (`list_scripts` only reports `draft_path` when it differs). + // Returning early on the path alone would drop the new summary silently. + if new_path == path && req.summary.is_none() { + return Ok("unchanged".to_string()); + } + require_can_write_path(&authed, &db, &user_db, &w_id, kind, path).await?; + if new_path != path { + require_can_write_path(&authed, &db, &user_db, &w_id, kind, new_path).await?; + } + + if let Some(table) = kind.deployed_table() { + // `table` is from the closed `deployed_table()` enum, never user input. + // Archived and soft-deleted rows keep sitting at their path — a script + // move archives its parent in place — so an existence check that counted + // them would refuse a move away from, or into, a path nothing occupies. + // `create_script_internal` resolves its own path clashes the same way. + let archived_filter = if table == "script" { + " AND NOT archived AND NOT deleted" + } else { + "" + }; + let query = format!( + "SELECT 1 FROM {table} WHERE path = $1 AND workspace_id = $2{archived_filter} LIMIT 1" + ); + let mut tx = user_db.clone().begin(&authed).await?; + let deployed_at_old = sqlx::query_scalar::<_, i32>(&query) + .bind(path) + .bind(&w_id) + .fetch_optional(&mut *tx) + .await?; + let deployed_at_new = sqlx::query_scalar::<_, i32>(&query) + .bind(new_path) + .bind(&w_id) + .fetch_optional(&mut *tx) + .await?; + tx.commit().await?; + if deployed_at_old.is_some() { + return Err(Error::BadRequest(format!( + "'{path}' is deployed — move it from its editor so schedules and triggers follow" + ))); + } + if deployed_at_new.is_some() { + return Err(Error::BadRequest(format!( + "'{new_path}' already has a deployed item — moving there would turn this into a draft on top of it" + ))); + } + } + + // A NUL in the summary reaches Postgres as a raw byte and fails the statement with + // an encoding error, so it is dropped here the way `strip_json_nul` drops one from a + // draft value. `json` cannot store it either. + let summary = req.summary.as_ref().map(|s| s.replace('\0', "")); + + // A classic app and a raw app share the `app` table, so a draft of either kind + // occupies the destination for both: deploying there deletes the caller's drafts + // of both kinds, taking the item that lost the collision with it. + let collision_typs: Vec<&str> = match kind { + UserDraftItemKind::App | UserDraftItemKind::RawApp => vec![ + UserDraftItemKind::App.as_str(), + UserDraftItemKind::RawApp.as_str(), + ], + _ => vec![kind.as_str()], + }; + + // One transaction with the move record, so a save addressed to the old path + // never sees the row gone without knowing where it went. + let mut tx = db.begin().await?; + let moved = sqlx::query_scalar!( + r#"UPDATE draft + SET path = $3, + -- Both path keys, not just the typed one: the editors mirror the + -- typed path into the other while it differs from the row's path, + -- and the loaders prefer the mirror — left naming the old location + -- it un-does this move on the next save. `create_missing = false` + -- on both, so a draft carrying only one keeps only one. + value = to_json( + jsonb_set( + jsonb_set( + CASE WHEN $7::text IS NULL THEN to_jsonb(value) + ELSE jsonb_set(to_jsonb(value), ARRAY['summary'], to_jsonb($7::text)) + END, + ARRAY[$5::text], to_jsonb($3::text), false + ), + ARRAY[$8::text], to_jsonb($3::text), false + ) + ) + WHERE workspace_id = $1 + AND path = $2 + AND typ = $4 + AND email = $6 + -- A pre-sanitizer NUL escape makes `to_jsonb` raise 22P05. Excluded + -- here so the statement can't 500; reported below instead. Unlike the + -- passive carry, rewriting the value IS this operation, so skipping it + -- silently would move the row and leave its typed path stale. + AND position(chr(92) || 'u0000' in replace(value::text, chr(92) || chr(92), '')) = 0 + -- Skipped on a summary-only edit, where the "target" row is this + -- row and the guard would refuse the update against itself. + AND ($2 = $3 OR NOT EXISTS ( + SELECT 1 FROM draft o + WHERE o.workspace_id = $1 AND o.path = $3 AND o.typ::text = ANY($9::text[]) + -- Of this kind only the caller's own row and the legacy one collide: + -- teammates' drafts of one item share its path by design, but a deploy + -- there wipes those two together, so a second would discard edits the + -- caller never saw. The other app kind is a different item on the same + -- deployed path, so it collides whoever owns it. + AND (o.typ <> $4 OR o.email = $6 OR o.email IS NULL) + )) + RETURNING id"#, + &w_id, + path, + new_path, + kind as UserDraftItemKind, + typed_field, + &authed.email, + summary, + mirror_field, + &collision_typs as &[&str], + ) + .fetch_optional(&mut *tx) + .await?; + if moved.is_some() && new_path != path { + windmill_common::user_drafts::record_draft_move( + &mut tx, + &w_id, + &[kind], + path, + new_path, + Some(&authed.email), + ) + .await?; + } + tx.commit().await?; + + if moved.is_none() { + let row = sqlx::query!( + r#"SELECT + -- The guard's own predicate, ordered own row, then legacy, then another + -- user's other-kind row: each needs a different remedy, and a bare + -- LIMIT 1 would name an arbitrary one. + (SELECT typ::text FROM draft WHERE workspace_id = $1 AND path = $3 + AND typ::text = ANY($6::text[]) + AND (typ <> $2 OR email = $4 OR email IS NULL) + ORDER BY CASE WHEN email = $4 THEN 0 WHEN email IS NULL THEN 1 ELSE 2 END + LIMIT 1) as "at_target", + EXISTS(SELECT 1 FROM draft WHERE workspace_id = $1 AND path = $3 + AND typ::text = ANY($6::text[]) AND email = $4) as "at_target_own!", + EXISTS(SELECT 1 FROM draft WHERE workspace_id = $1 AND path = $3 + AND typ::text = ANY($6::text[]) AND email IS NULL) as "at_target_legacy!", + EXISTS(SELECT 1 FROM draft WHERE workspace_id = $1 AND path = $5 + AND typ = $2 AND email = $4 + AND position(chr(92) || 'u0000' in replace(value::text, chr(92) || chr(92), '')) > 0 + ) as "poisoned!", + EXISTS(SELECT 1 FROM draft WHERE workspace_id = $1 AND path = $5 + AND typ = $2 AND email IS NULL) as "legacy!" "#, + &w_id, + kind as UserDraftItemKind, + new_path, + &authed.email, + path, + &collision_typs as &[&str], + ) + .fetch_one(&db) + .await?; + return Err(Error::BadRequest(if row.poisoned { + // This endpoint also serves a summary-only edit, so name the operation + // the caller actually asked for rather than always saying "moved". + let attempted = if new_path == path { "updated" } else { "moved" }; + format!( + "'{path}' contains a NUL character and predates the sanitizer, so it cannot be \ + {attempted}. Reopen it, re-save to rewrite it cleanly, then retry." + ) + } else if row.legacy { + // The home list synthesizes a draft-only row for the legacy draft with the + // caller's own name on it, so this is reachable from the row menu. Only an + // admin can claim or discard that row, and only from the drafts page. + format!( + "'{path}' is a legacy workspace draft with no owner, so it cannot be moved. \ + A workspace admin can claim or discard it on the Review & deploy drafts page." + ) + } else if let Some(occupant) = row.at_target.filter(|_| new_path != path) { + // Naming the kind matters for the app pair: a classic-app draft refusing a + // raw-app move is invisible in the raw-app list the caller is looking at. + let occupant = occupant.replace('_', " "); + if row.at_target_own { + format!("You already have a draft at '{new_path}' ({occupant})") + } else if row.at_target_legacy { + // An ownerless row the caller cannot clear themselves, so send them to + // the one place it can be resolved rather than to "discard your draft". + format!( + "A legacy workspace draft with no owner is already at '{new_path}' \ + ({occupant}). A workspace admin can claim or discard it on the Review & \ + deploy drafts page." + ) + } else { + // The other app kind, owned by someone else: one deployed path cannot hold + // both, so this is the other item's path, not a teammate's copy of this one. + format!( + "'{new_path}' holds another user's {occupant} draft, and an app and a raw \ + app cannot share a path. Pick another path, or ask them to move or \ + discard theirs." + ) + } + } else { + format!("You have no draft at '{path}'") + })); + } + + if new_path == path { + return Ok(format!("updated draft {path}")); + } + Ok(format!("moved draft {path} to {new_path}")) +} + #[derive(Deserialize, Debug)] #[serde(rename_all = "snake_case")] pub enum MigrateLegacyDraftAction { @@ -504,12 +946,12 @@ async fn migrate_legacy_draft( r#"WITH legacy AS ( DELETE FROM draft WHERE workspace_id = $1 AND path = $2 AND typ = $3 AND email IS NULL - RETURNING value + RETURNING value, base ) - INSERT INTO draft (workspace_id, email, path, typ, value, created_at) - SELECT $1, $4, $2, $3, value, now() FROM legacy + INSERT INTO draft (workspace_id, email, path, typ, value, created_at, base) + SELECT $1, $4, $2, $3, value, now(), base FROM legacy ON CONFLICT (workspace_id, path, typ, email) WHERE email IS NOT NULL - DO UPDATE SET value = EXCLUDED.value, created_at = now() + DO UPDATE SET value = EXCLUDED.value, created_at = now(), base = EXCLUDED.base RETURNING 1 as "one!""#, &w_id, path, @@ -767,8 +1209,8 @@ async fn require_can_write_path( /// Resolves to `Ok(())` if `authed` can read at `path`. Three layers: /// 1. admin → always. -/// 2. Path-prefix match against own `u/{username}` or any folder in -/// `authed.folders` (the precomputed read set, with groups + direct +/// 2. Path-prefix match against own `u/{username}`, a group in `authed.groups`, or any +/// folder in `authed.folders` (the precomputed read set, with groups + direct /// grants already factored in). /// 3. RLS-aware `SELECT 1` against the backing table — covers item-level /// extra_perms grants that bypass folder/owner checks. @@ -793,6 +1235,10 @@ async fn require_can_read_path( if parts.len() >= 2 { match parts[0] { "u" if parts[1] == authed.username => return Ok(()), + // As `require_can_write_path` and the `see_member` RLS policy read it: a + // draft-only `g/` path has no row for the probe below to fall back on, so + // without this a member cannot see a draft their group owns. + "g" if authed.groups.iter().any(|g| g == parts[1]) => return Ok(()), "f" => { let folder = parts[1]; if authed.folders.iter().any(|(name, _, _)| name == folder) { diff --git a/backend/windmill-api/src/jobs.rs b/backend/windmill-api/src/jobs.rs index 74f24ca415..052c9df4bb 100644 --- a/backend/windmill-api/src/jobs.rs +++ b/backend/windmill-api/src/jobs.rs @@ -7226,7 +7226,10 @@ pub async fn run_workflow_as_code( ) -> error::Result<(StatusCode, String)> { #[cfg(feature = "enterprise")] check_license_key_valid().await?; - check_tag_available_for_workspace(&db, &w_id, &run_query.tag, &authed).await?; + let mut extra = HashMap::new(); + extra.insert(ENTRYPOINT_OVERRIDE.to_string(), to_raw_value(&entrypoint)); + let args = PushArgs { args: &task.args.unwrap_or_else(HashMap::new), extra: Some(extra) }; + check_tag_available_for_workspace(&db, &w_id, &run_query.tag, &args, &authed).await?; check_scopes(&authed, || format!("jobs:run"))?; if !is_valid_entrypoint_name(&entrypoint) { @@ -7320,10 +7323,6 @@ pub async fn run_workflow_as_code( i += 1; } - let mut extra = HashMap::new(); - extra.insert(ENTRYPOINT_OVERRIDE.to_string(), to_raw_value(&entrypoint)); - - let args = PushArgs { args: &task.args.unwrap_or_else(HashMap::new), extra: Some(extra) }; let scheduled_for = run_query.get_scheduled_for(&db).await?; let tag = run_query.tag.clone().or(tag).or(Some(job.tag)); @@ -7360,7 +7359,7 @@ pub async fn run_workflow_as_code( tx, &w_id, job_payload, - PushArgs { args: &args.args, extra: args.extra }, + args, authed.display_username(), email, permissioned_as, @@ -7646,7 +7645,8 @@ pub async fn run_wait_result_job_by_path_get( .await?; let tag = run_query.tag.clone().or(tag); - check_tag_available_for_workspace(&db, &w_id, &tag, &authed).await?; + let push_args = PushArgs { args: &args.args, extra: args.extra }; + check_tag_available_for_workspace(&db, &w_id, &tag, &push_args, &authed).await?; let (email, permissioned_as, push_authed, tx) = if let Some(on_behalf_of) = on_behalf_authed.as_ref() { @@ -7670,7 +7670,7 @@ pub async fn run_wait_result_job_by_path_get( tx, &w_id, job_payload, - PushArgs { args: &args.args, extra: args.extra }, + push_args, authed.display_username(), email, permissioned_as, @@ -7791,7 +7791,8 @@ pub async fn run_wait_result_script_by_path_internal( .await?; let tag = run_query.tag.clone().or(tag); - check_tag_available_for_workspace(&db, &w_id, &tag, &authed).await?; + let push_args = PushArgs { args: &args.args, extra: args.extra }; + check_tag_available_for_workspace(&db, &w_id, &tag, &push_args, &authed).await?; let (email, permissioned_as, push_authed, tx) = if let Some(on_behalf_of) = on_behalf_of.as_ref() { @@ -7815,7 +7816,7 @@ pub async fn run_wait_result_script_by_path_internal( tx, &w_id, job_payload, - PushArgs { args: &args.args, extra: args.extra }, + push_args, authed.display_username(), email, permissioned_as, @@ -7904,7 +7905,8 @@ pub async fn run_wait_result_script_by_hash( check_scopes(&authed, || format!("jobs:run:scripts:{path}"))?; let tag = run_query.tag.clone().or(tag); - check_tag_available_for_workspace(&db, &w_id, &tag, &authed).await?; + let push_args = PushArgs { args: &args.args, extra: args.extra }; + check_tag_available_for_workspace(&db, &w_id, &tag, &push_args, &authed).await?; let (email, permissioned_as, push_authed, tx) = if let Some(obo) = on_behalf_of.as_ref() { ( @@ -7940,7 +7942,7 @@ pub async fn run_wait_result_script_by_hash( && has_preprocessor.unwrap_or(false), labels, }, - PushArgs { args: &args.args, extra: args.extra }, + push_args, authed.display_username(), email, permissioned_as, @@ -8465,7 +8467,6 @@ async fn run_preview_script( require_path_read_access_for_preview(&authed, &preview.path)?; let scheduled_for = run_query.get_scheduled_for(&db).await?; let tag = run_query.tag.clone().or(preview.tag.clone()); - check_tag_available_for_workspace(&db, &w_id, &tag, &authed).await?; let tx = PushIsolationLevel::Isolated(user_db.clone(), authed.clone().into()); let preview_args = preview.args.unwrap_or_default(); @@ -8484,6 +8485,7 @@ async fn run_preview_script( } let extra = if extra.is_empty() { None } else { Some(extra) }; let push_args = PushArgs { extra, args: &preview_args }; + check_tag_available_for_workspace(&db, &w_id, &tag, &push_args, &authed).await?; let (uuid, tx) = push( &db, @@ -8723,7 +8725,12 @@ fn register_potential_assets_on_inline_execution( .as_ref() .and_then(|args| args.get("database")) .map(|v| v.get().trim_matches('"')) - .and_then(|dt| dt.strip_prefix("datatable://")); + .and_then(|dt| dt.strip_prefix("datatable://")) + // `?role=` picks the connection, not the data table. Anything else after a `?` may be + // part of a name stored before names were restricted, so it stays. + .map(|dt| { + windmill_common::workspaces::parse_datatable_ref(dt).map_or(dt, |(name, _)| name) + }); if let Some(datatable) = datatable { let re = regex::Regex::new(r#"SET search_path TO "([^"]+)";"#).unwrap(); let (schema, content) = if let Some(captures) = re.captures(&preview.content) { @@ -8840,7 +8847,6 @@ async fn run_bundle_preview_script( let scheduled_for = run_query.get_scheduled_for(&db).await?; let tag = run_query.tag.clone().or(preview.tag.clone()); - check_tag_available_for_workspace(&db, &w_id, &tag, &authed).await?; let ltx = PushIsolationLevel::Isolated(user_db.clone(), authed.clone().into()); let args = preview.args.unwrap_or_default(); @@ -8855,6 +8861,7 @@ async fn run_bundle_preview_script( m }); let push_args = PushArgs { extra, args: &args }; + check_tag_available_for_workspace(&db, &w_id, &tag, &push_args, &authed).await?; is_tar = match preview.kind { Some(PreviewKind::Tarbundle) => true, @@ -9506,7 +9513,6 @@ async fn run_preview_flow_job( require_path_read_access_for_preview(&authed, &raw_flow.path)?; let scheduled_for = run_query.get_scheduled_for(&db).await?; let tag = run_query.tag.clone().or(raw_flow.tag.clone()); - check_tag_available_for_workspace(&db, &w_id, &tag, &authed).await?; let tx = PushIsolationLevel::Isolated(user_db.clone(), authed.clone().into()); let chat_input_enabled = raw_flow.value.chat_input_enabled.unwrap_or(false); @@ -9524,6 +9530,8 @@ async fn run_preview_flow_job( to_raw_value(temp_script_refs), ); } + check_tag_available_for_workspace(&db, &w_id, &tag, &PushArgs::from(&flow_args), &authed) + .await?; let (uuid, mut tx) = push( &db, @@ -9769,11 +9777,6 @@ async fn run_dynamic_select( } } - // Same tag-permission gate a normal run gets (run_flow / push_script_job_by_path_into_queue): - // a caller allowed to read the flow must still be allowed to use its worker tag. No-op for - // inline (tag is None); the script branch checked this inside its helper and returned above. - check_tag_available_for_workspace(&db, &w_id, &tag, &authed).await?; - // Invoke the dyn-select entrypoint instead of `main`. let mut args = request.args.unwrap_or_default(); args.insert( @@ -9781,6 +9784,11 @@ async fn run_dynamic_select( serde_json::value::to_raw_value(&request.entrypoint_function)?, ); + // Same tag-permission gate a normal run gets (run_flow / push_script_job_by_path_into_queue): + // a caller allowed to read the flow must still be allowed to use its worker tag. No-op for + // inline (tag is None); the script branch checked this inside its helper and returned above. + check_tag_available_for_workspace(&db, &w_id, &tag, &PushArgs::from(&args), &authed).await?; + let scheduled_for = run_query.get_scheduled_for(&db).await?; let tx = PushIsolationLevel::Isolated(user_db.clone(), authed.clone().into()); @@ -9915,8 +9923,9 @@ pub async fn run_job_by_hash_inner( } let scheduled_for = run_query.get_scheduled_for(&db).await?; let tag = run_query.tag.clone().or(tag); + let push_args = PushArgs { args: &args.args, extra: args.extra }; - check_tag_available_for_workspace(&db, &w_id, &tag, &authed).await?; + check_tag_available_for_workspace(&db, &w_id, &tag, &push_args, &authed).await?; let (email, permissioned_as, push_authed, tx) = if let Some(obo) = on_behalf_of.as_ref() { ( @@ -9952,7 +9961,7 @@ pub async fn run_job_by_hash_inner( && has_preprocessor.unwrap_or(false), labels, }, - PushArgs { args: &args.args, extra: args.extra }, + push_args, authed.display_username(), email, permissioned_as, diff --git a/backend/windmill-api/src/lib.rs b/backend/windmill-api/src/lib.rs index 0fd5ac31b5..d6b29a8501 100644 --- a/backend/windmill-api/src/lib.rs +++ b/backend/windmill-api/src/lib.rs @@ -731,6 +731,12 @@ pub async fn run_server( path_autocomplete::workspaced_service(), ) .nest("/raw_apps", raw_apps::workspaced_service()) + .nest( + "/remote_deploy", + windmill_api_workspaces::remote_deploy::workspaced_service( + request_size_limit * 5, + ), + ) // CORS so the opaque-origin app iframe can read // resources/list, resources/type/* with a scoped token. .nest( diff --git a/backend/windmill-api/src/live_migrations.rs b/backend/windmill-api/src/live_migrations.rs index 4c6f5e245c..334ee340a8 100644 --- a/backend/windmill-api/src/live_migrations.rs +++ b/backend/windmill-api/src/live_migrations.rs @@ -6,8 +6,9 @@ * LICENSE-AGPL for a copy of the license. */ -use sqlx::Postgres; -use windmill_common::error::Error; +use sqlx::{PgConnection, Postgres}; +use tokio::task::JoinHandle; +use windmill_common::{db::DB, error::Error}; use crate::db::CustomMigrator; use sqlx::migrate::Migrate; @@ -121,3 +122,294 @@ async fn fix_flow_versioning_migration(migrator: &mut CustomMigrator) -> Result< } Ok(()) } + +// Held for the whole background run so only one server does it at a time. +const BACKGROUND_MIGRATIONS_LOCK_ID: i64 = 4_931_072_518_336_401; + +/// Schema changes too slow to hold server startup for, run by one server at a time after the +/// sqlx migrations. Each step is recorded in `windmill_migrations` once done; a step interrupted +/// by a restart or an error starts over on the next start and must resume safely. +pub fn spawn_background_migrations( + db: DB, + mut killpill_rx: tokio::sync::broadcast::Receiver<()>, +) -> JoinHandle<()> { + tokio::spawn(async move { + tokio::select! { + r = run_background_migrations(&db) => { + if let Err(err) = r { + tracing::error!("Background migrations stopped, retrying on the next start: {err:#}"); + } + } + _ = killpill_rx.recv() => { + tracing::info!("Killpill received, stopping background migrations"); + } + } + }) +} + +async fn run_background_migrations(db: &DB) -> Result<(), Error> { + // Detached so the pool's 5min statement_timeout, lifted here for the index builds, never + // comes back with this connection; closing it also releases the advisory lock. + let mut conn = db.acquire().await?.detach(); + conn.execute("SET statement_timeout = 0").await?; + let locked = sqlx::query_scalar::<_, bool>("SELECT pg_try_advisory_lock($1)") + .bind(BACKGROUND_MIGRATIONS_LOCK_ID) + .fetch_one(&mut conn) + .await?; + if !locked { + return Ok(()); + } + + for step in [AUDIT_OPERATION_INDEX, RETIRE_LEGACY_AUDIT] { + if background_migration_done(&mut conn, step).await? { + continue; + } + // Steps bound the lock waits that would queue audit inserts behind them with + // lock_timeout, and resume where they stopped, so a lock timeout is retried here. + let mut attempt = 1; + loop { + let run = match step { + AUDIT_OPERATION_INDEX => create_audit_operation_index(&mut conn).await, + RETIRE_LEGACY_AUDIT => retire_legacy_audit_table(&mut conn).await, + _ => unreachable!("background migration {step} has no step function"), + }; + match run { + Ok(()) => break, + Err(err) if attempt < 10 && is_lock_timeout(&err) => { + tracing::warn!( + "Background migration {step} timed out on a lock, retrying in 30s: {err:#}" + ); + attempt += 1; + tokio::time::sleep(std::time::Duration::from_secs(30)).await; + } + Err(err) => return Err(err), + } + } + mark_background_migration_done(&mut conn, step).await?; + } + Ok(()) +} + +const AUDIT_OPERATION_INDEX: &str = "audit_partitioned_workspace_operation_index"; +const RETIRE_LEGACY_AUDIT: &str = "retire_legacy_audit_table"; + +// Short on purpose: a statement waiting for its lock is also a wait for every audit insert +// queued behind it. +const STEP_LOCK_TIMEOUT: &str = "SET lock_timeout = '1s'"; + +fn is_lock_timeout(err: &Error) -> bool { + matches!(err, Error::SqlErr { error: sqlx::Error::Database(db_err), .. } + if db_err.code().as_deref() == Some("55P03")) +} + +async fn background_migration_done(conn: &mut PgConnection, name: &str) -> Result { + Ok(sqlx::query_scalar::<_, bool>( + "SELECT EXISTS(SELECT 1 FROM windmill_migrations WHERE name = $1)", + ) + .bind(name) + .fetch_one(conn) + .await?) +} + +async fn mark_background_migration_done(conn: &mut PgConnection, name: &str) -> Result<(), Error> { + sqlx::query("INSERT INTO windmill_migrations (name) VALUES ($1) ON CONFLICT DO NOTHING") + .bind(name) + .execute(conn) + .await?; + tracing::info!("Background migration {name} done"); + Ok(()) +} + +const AUDIT_OPERATION_INDEX_KEY: &str = r#"(workspace_id, operation, id DESC, "timestamp")"#; + +/// A plain `CREATE INDEX` on the partitioned table holds a SHARE lock on every partition until the +/// whole build ends, blocking the audit insert each job push makes in its own transaction. Each +/// partition is built CONCURRENTLY instead and attached to a parent created `ON ONLY`, which turns +/// valid once all partitions are attached. Partitions created later get the index from the parent. +async fn create_audit_operation_index(conn: &mut PgConnection) -> Result<(), Error> { + // Metadata only, but it waits for every open transaction that wrote an audit row, and every + // audit insert (so every job push) queues behind it meanwhile. Give up early instead. + conn.execute(STEP_LOCK_TIMEOUT).await?; + let created = conn + .execute( + format!( + "CREATE INDEX IF NOT EXISTS ix_audit_partitioned_workspace_operation \ + ON ONLY audit_partitioned {AUDIT_OPERATION_INDEX_KEY}" + ) + .as_str(), + ) + .await; + conn.execute("RESET lock_timeout").await?; + created?; + let partitions: Vec = sqlx::query_scalar( + "SELECT c.relname::text FROM pg_inherits p JOIN pg_class c ON c.oid = p.inhrelid + WHERE p.inhparent = 'audit_partitioned'::regclass + AND NOT EXISTS ( + SELECT 1 FROM pg_inherits ip JOIN pg_index i ON i.indexrelid = ip.inhrelid + WHERE ip.inhparent = 'ix_audit_partitioned_workspace_operation'::regclass + AND i.indrelid = c.oid) + ORDER BY c.relname DESC", + ) + .fetch_all(&mut *conn) + .await?; + let quote = |name: &str| format!("\"{}\"", name.replace('"', "\"\"")); + for partition in partitions { + let index = quote(&format!( + "{partition}_workspace_id_operation_id_timestamp_idx" + )); + tracing::info!("Building ix_audit_partitioned_workspace_operation on {partition}"); + // An interrupted CONCURRENTLY build leaves an invalid index under this name. + conn.execute(format!("DROP INDEX CONCURRENTLY IF EXISTS {index}").as_str()) + .await?; + conn.execute( + format!( + "CREATE INDEX CONCURRENTLY {index} ON {} {AUDIT_OPERATION_INDEX_KEY}", + quote(&partition) + ) + .as_str(), + ) + .await?; + conn.execute( + format!( + "ALTER INDEX ix_audit_partitioned_workspace_operation ATTACH PARTITION {index}" + ) + .as_str(), + ) + .await?; + } + Ok(()) +} + +const AUDIT_COLUMNS: &str = + "workspace_id, id, timestamp, username, operation, action_kind, resource, parameters, email, span"; + +/// Moves the last 30 days of the pre-partitioning `audit` table into daily partitions and drops +/// it with anything older. An empty `audit` view takes its place for servers still running an +/// older version, which read it in a `UNION ALL` with `audit_partitioned`. +async fn retire_legacy_audit_table(conn: &mut PgConnection) -> Result<(), Error> { + let is_table = sqlx::query_scalar::<_, bool>( + "SELECT EXISTS(SELECT 1 FROM pg_class WHERE oid = to_regclass('audit') AND relkind = 'r')", + ) + .fetch_one(&mut *conn) + .await?; + if !is_table { + return Ok(()); + } + // Creating a partition and re-owning the sequence queue every audit insert (so every job + // push) behind them while they wait for their own lock, e.g. on a long-running reader, and + // the DROP queues older servers' reads the same way. Give up early and retry instead. + conn.execute(STEP_LOCK_TIMEOUT).await?; + + let days: Vec = sqlx::query_scalar( + "SELECT DISTINCT timestamp::date FROM audit + WHERE timestamp > now() - interval '30 days' ORDER BY 1", + ) + .fetch_all(&mut *conn) + .await?; + for day in days { + let next = day + chrono::Duration::days(1); + // Created outside any transaction: a partition created inside one keeps a lock on + // audit_partitioned that blocks every job push's audit insert until it commits. + conn.execute( + format!( + "CREATE TABLE IF NOT EXISTS \"audit_{}\" PARTITION OF audit_partitioned \ + FOR VALUES FROM ('{day}') TO ('{next}')", + day.format("%Y%m%d") + ) + .as_str(), + ) + .await?; + // One statement per day, so a row is always in exactly one of the two tables. + let moved = sqlx::query(&format!( + "WITH moved AS ( + DELETE FROM audit WHERE timestamp >= $1::date AND timestamp < $1::date + 1 + RETURNING {AUDIT_COLUMNS} + ) + INSERT INTO audit_partitioned ({AUDIT_COLUMNS}) SELECT {AUDIT_COLUMNS} FROM moved" + )) + .bind(day) + .execute(&mut *conn) + .await?; + tracing::info!( + "Moved {} legacy audit rows of {day} into audit_partitioned", + moved.rows_affected() + ); + } + + // audit_partitioned draws its ids from this sequence, which dropping its owner would drop. + // Not in the transaction below: its lock stops every insert's nextval until commit, and the + // DROP can wait there on readers of the old table. + conn.execute("ALTER SEQUENCE audit_id_seq OWNED BY audit_partitioned.id") + .await?; + let mut tx = conn.begin().await?; + // Servers older than audit partitioning still write to the table during a rolling upgrade: + // rows they added after their day was moved above are swept up under the lock. + tx.execute("LOCK TABLE audit IN ACCESS EXCLUSIVE MODE") + .await?; + tx.execute( + format!( + "WITH moved AS ( + DELETE FROM audit WHERE timestamp > now() - interval '30 days' + RETURNING {AUDIT_COLUMNS} + ) + INSERT INTO audit_partitioned ({AUDIT_COLUMNS}) SELECT {AUDIT_COLUMNS} FROM moved" + ) + .as_str(), + ) + .await?; + tx.execute("DROP TABLE audit").await?; + tx.execute( + format!("CREATE VIEW audit AS SELECT {AUDIT_COLUMNS} FROM audit_partitioned WHERE false") + .as_str(), + ) + .await?; + tx.execute("GRANT ALL ON audit TO windmill_user, windmill_admin") + .await?; + tx.commit().await?; + conn.execute("RESET lock_timeout").await?; + tracing::info!("Retired the legacy audit table"); + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[sqlx::test(migrations = "../migrations")] + async fn retire_legacy_audit_table_keeps_the_last_30_days(db: DB) -> anyhow::Result<()> { + sqlx::query( + "INSERT INTO audit (workspace_id, id, timestamp, username, operation, action_kind) + VALUES ('w', -2, now() - interval '3 days', 'u', 'recent', 'execute'), + ('w', -1, now() - interval '60 days', 'u', 'old', 'execute')", + ) + .execute(&db) + .await?; + + retire_legacy_audit_table(&mut *db.acquire().await?).await?; + + let moved: Vec<(i64, String)> = sqlx::query_as( + "SELECT id, operation::text FROM audit_partitioned WHERE workspace_id = 'w'", + ) + .fetch_all(&db) + .await?; + assert_eq!(moved, vec![(-2, "recent".to_string())]); + + // The id sequence outlives the table that owned it. + sqlx::query( + "INSERT INTO audit_partitioned (workspace_id, username, operation, action_kind) + VALUES ('w', 'u', 'after', 'execute')", + ) + .execute(&db) + .await?; + + // Servers on an older version still read `audit` in a UNION with audit_partitioned. + let seen: i64 = sqlx::query_scalar( + "SELECT count(*) FROM (SELECT * FROM audit_partitioned UNION ALL SELECT * FROM audit) a + WHERE workspace_id = 'w'", + ) + .fetch_one(&db) + .await?; + assert_eq!(seen, 2); + Ok(()) + } +} diff --git a/backend/windmill-api/src/raw_apps.rs b/backend/windmill-api/src/raw_apps.rs index d7bc3bc6c0..35e4f693a4 100644 --- a/backend/windmill-api/src/raw_apps.rs +++ b/backend/windmill-api/src/raw_apps.rs @@ -5,7 +5,10 @@ * Please see the included NOTICE for copyright information and * LICENSE-AGPL for a copy of the license. */ -use crate::{db::ApiAuthed, utils::check_scopes}; +use crate::{ + db::ApiAuthed, + utils::{build_scope_path_filter, check_scopes, ScopePathFilter}, +}; use axum::{ body::Body, extract::{Extension, Json, Path, Query}, @@ -103,6 +106,31 @@ async fn list_apps( } } + // In the WHERE, not a retain after the fetch: the result is paginated, and a + // post-fetch filter would return short pages and let a page's size report how + // many raw apps the token may not read. One `?` per term, since a chained `.bind` + // substitutes into a `?` inside the value an earlier one inserted; `starts_with`, + // since LIKE reads a `_` in the path as a wildcard. + if let ScopePathFilter::Restricted { exact, prefix } = + build_scope_path_filter(&authed, "raw_apps", "read") + { + let terms: Vec = exact + .iter() + .chain(&prefix) + .map(|p| "app.path = ?".bind(p)) + .chain( + prefix + .iter() + .map(|p| "starts_with(app.path, ?)".bind(&format!("{p}/"))), + ) + .collect(); + sqlb.and_where(if terms.is_empty() { + "false".to_string() + } else { + format!("({})", terms.join(" OR ")) + }); + } + let sql = sqlb.sql().map_err(|e| Error::internal_err(e.to_string()))?; let mut tx = user_db.begin(&authed).await?; let rows = sqlx::query_as::<_, ListableApp>(&sql) diff --git a/backend/windmill-api/src/tracing_init.rs b/backend/windmill-api/src/tracing_init.rs index 56adeee184..d99a7e5907 100644 --- a/backend/windmill-api/src/tracing_init.rs +++ b/backend/windmill-api/src/tracing_init.rs @@ -14,6 +14,7 @@ use axum::response::Response as AxumResponse; use hyper::Response; use tower_http::trace::{MakeSpan, OnFailure, OnResponse}; use uuid::Uuid; +use windmill_api_workspaces::remote_deploy::RedactedUri; use windmill_common::log_context::{with_log_context, LogContext}; lazy_static::lazy_static! { @@ -86,7 +87,7 @@ impl MakeSpan for MyMakeSpan { tracing::error_span!( "request", method = %request.method(), - uri = %request.uri(), + uri = %RedactedUri(request.uri()), username = field::Empty, workspace_id = field::Empty, traceId = tracing_id, @@ -113,7 +114,7 @@ pub async fn log_context_middleware(request: Request, next: Next) -> AxumRespons let ctx = LogContext { method: Some(request.method().to_string()), - uri: Some(request.uri().to_string()), + uri: Some(RedactedUri(request.uri()).to_string()), trace_id, ..Default::default() }; diff --git a/backend/windmill-api/src/workspaces.rs b/backend/windmill-api/src/workspaces.rs index a0a27eab4c..186cfbc291 100644 --- a/backend/windmill-api/src/workspaces.rs +++ b/backend/windmill-api/src/workspaces.rs @@ -108,6 +108,7 @@ async fn edit_copilot_config( } ai_config.validate_model_pricing()?; + ai_config.validate_token_limits()?; ai_config.validate_sessions_retention()?; let mut tx = db.begin().await?; diff --git a/backend/windmill-api/src/workspaces_export.rs b/backend/windmill-api/src/workspaces_export.rs index 4f6a6d9efb..e29113ecd2 100644 --- a/backend/windmill-api/src/workspaces_export.rs +++ b/backend/windmill-api/src/workspaces_export.rs @@ -1425,9 +1425,17 @@ pub(crate) async fn tarball_workspace( use strum::IntoEnumIterator; for service_name in ServiceName::iter() { - let native_triggers = - list_native_triggers(&mut *tx, &w_id, service_name, None, None, None, None) - .await?; + let native_triggers = list_native_triggers( + &mut *tx, + &w_id, + service_name, + None, + None, + None, + None, + windmill_api_auth::ScopePathFilter::AllowAll, + ) + .await?; // Native triggers (Nextcloud, Google Drive, GitHub) are never // cloned into a fork — a fork only has one if its owner created diff --git a/backend/windmill-common/src/auth.rs b/backend/windmill-common/src/auth.rs index c12d62f206..86b7f173ac 100644 --- a/backend/windmill-common/src/auth.rs +++ b/backend/windmill-common/src/auth.rs @@ -20,7 +20,8 @@ use crate::{ /// Whether `label` denotes a user-created token rather than a system token /// (`session`, `guest_session`, `ephemeral*`, `debugger-token`, `mcp-oauth-*`, -/// `embed_app:*`, `sdk_app:*`, `impersonation:*`, `cli-login:*`). System-token labels are load-bearing — +/// `embed_app:*`, `sdk_app:*`, `impersonation:*`, `cli-login:*`, `remote-deploy:*`). +/// System-token labels are load-bearing — /// session cleanup, super_admin propagation, expiry notifications and username overrides /// all key off them — so they must not be user-editable. `None` (no label) is treated as /// a user token. @@ -48,6 +49,7 @@ pub fn is_user_token(label: Option<&str>) -> bool { && !l.starts_with(RAW_APP_SDK_TOKEN_LABEL_PREFIX) && !l.starts_with("impersonation:") && !l.starts_with(CLI_LOGIN_TOKEN_LABEL_PREFIX) + && !l.starts_with(REMOTE_DEPLOY_TOKEN_LABEL_PREFIX) } } } @@ -73,6 +75,14 @@ pub const RAW_APP_SDK_TOKEN_LABEL_PREFIX: &str = "sdk_app:"; /// `/users/tokens/create`. pub const CLI_LOGIN_TOKEN_LABEL_PREFIX: &str = "cli-login:"; +/// Label prefix, followed by the host of the instance deploying with it, of the token the +/// `/user/remote_deploy_authorize` page mints for another Windmill instance. Reserved in +/// [`is_user_token`], whose SQL and frontend mirrors spell it out: it is renewed by connecting +/// again from the deploying instance, which reports this one rejecting it, so an expiry email or +/// alert here points at the wrong place. Not in [`is_server_minted_label`], since the page mints it +/// through `/users/tokens/create`. +pub const REMOTE_DEPLOY_TOKEN_LABEL_PREFIX: &str = "remote-deploy:"; + /// Whether `label` belongs to a namespace only the server mints, and which therefore must be /// rejected by `create_token`. Narrower than [`is_user_token`], which also drives label /// editability and expiry notifications and can afford to reserve more: `Ephemeral lsp token`, @@ -997,6 +1007,7 @@ mod tests { assert!(!is_user_token(Some("sdk_app:u/admin/raw app"))); assert!(!is_user_token(Some("impersonation:admin@windmill.dev"))); assert!(!is_user_token(Some("cli-login:admin"))); + assert!(!is_user_token(Some("remote-deploy:windmill.example.com"))); } #[test] diff --git a/backend/windmill-common/src/datatable_roles_oss.rs b/backend/windmill-common/src/datatable_roles_oss.rs index 057a739bd4..2a3f9dda48 100644 --- a/backend/windmill-common/src/datatable_roles_oss.rs +++ b/backend/windmill-common/src/datatable_roles_oss.rs @@ -15,7 +15,8 @@ use crate::error::Error; -/// What every roles path answers without the Enterprise Edition. +/// What every roles path answers without the Enterprise Edition. The frontend matches this exact +/// sentence (`datatableUsableRoles.ts`) to read the refusal as "not under roles": reword both. pub fn datatable_roles_unavailable() -> Error { Error::BadRequest("Data table roles are a Windmill Enterprise Edition feature".to_string()) } diff --git a/backend/windmill-common/src/guest_jwt.rs b/backend/windmill-common/src/guest_jwt.rs index 9f3e47d708..dc0407163d 100644 --- a/backend/windmill-common/src/guest_jwt.rs +++ b/backend/windmill-common/src/guest_jwt.rs @@ -167,7 +167,13 @@ pub fn jwk_algorithms(jwk: &Jwk) -> Option> { { return None; } - match (&jwk.algorithm, jwk.common.algorithm) { + let alg = crate::jwt::jwk_algorithm(jwk); + // A key that names an `alg` we cannot verify with (RSA-OAEP, ...) is not an alg-less + // key to be tried against the whole family: it is published for something else. + if alg.is_none() && jwk.common.key_algorithm.is_some() { + return None; + } + match (&jwk.algorithm, alg) { (AlgorithmParameters::RSA(_), Some(alg)) if RSA_ALGORITHMS.contains(&alg) => { Some(vec![alg]) } @@ -197,6 +203,9 @@ pub fn verify( let mut validation = Validation::new(algorithms[0]); validation.algorithms = algorithms.to_vec(); validation.validate_nbf = true; + // Audience is not part of the guest contract: issuers commonly stamp their own `aud`, + // and no accepted audience is configured, so an `aud` claim must not fail the token. + validation.validate_aud = false; let claims = jsonwebtoken::decode::(token, key, &validation) .map_err(|e| Error::NotAuthorized(format!("guest JWT refused: {e}")))? .claims; @@ -648,6 +657,12 @@ mod tests { assert_eq!(jwk_algorithms(&k), None); } + #[test] + fn a_key_naming_an_encryption_alg_is_refused() { + let k = jwk(serde_json::json!({"kty":"RSA","alg":"RSA-OAEP","n":"aa","e":"AQAB"})); + assert_eq!(jwk_algorithms(&k), None); + } + #[test] fn a_key_marked_for_encryption_is_refused() { let k = jwk(serde_json::json!({"kty":"RSA","use":"enc","n":"aa","e":"AQAB"})); @@ -789,6 +804,50 @@ mod tests { .is_err()); } + #[test] + fn a_token_carrying_an_audience_still_verifies() { + // jsonwebtoken validates `aud` by default when the token has one, which would refuse + // every issuer that stamps its own audience. + let key = jsonwebtoken::DecodingKey::from_jwk(&jwk(serde_json::json!({ + "kty": "EC", "crv": "P-256", "kid": "k1", "x": PUB1_X, "y": PUB1_Y + }))) + .unwrap(); + let token = jsonwebtoken::encode( + &jsonwebtoken::Header::new(Algorithm::ES256), + &serde_json::json!({ + "email": "g@example.com", + "workspace_id": "ws", + "app_path": "u/a/app", + "aud": "https://issuer.example/api", + "exp": jsonwebtoken::get_current_timestamp() + 600, + }), + &jsonwebtoken::EncodingKey::from_ec_pem(PRIV1.as_bytes()).unwrap(), + ) + .unwrap(); + verify(&token, &key, &[Algorithm::ES256], "ws").expect("aud is not validated"); + } + + #[test] + fn an_8192_bit_rsa_key_verifies() { + // The largest modulus real issuers use; a crypto backend capped at 4096 bits rejects + // every token signed with it while the key itself still loads. + const PRIVATE: &str = include_str!("../tests/fixtures/guest_jwt_rsa8192_private.pem"); + const PUBLIC: &str = include_str!("../tests/fixtures/guest_jwt_rsa8192_public.pem"); + let (key, algs) = decoding_key_from_pem(PUBLIC).expect("8192-bit SPKI loads"); + let token = jsonwebtoken::encode( + &jsonwebtoken::Header::new(Algorithm::RS256), + &serde_json::json!({ + "email": "g@example.com", + "workspace_id": "ws", + "app_path": "u/a/app", + "exp": jsonwebtoken::get_current_timestamp() + 600, + }), + &jsonwebtoken::EncodingKey::from_rsa_pem(PRIVATE.as_bytes()).unwrap(), + ) + .unwrap(); + verify(&token, &key, algs, "ws").expect("8192-bit RSA signature verifies"); + } + // A real RSA public key (SPKI). Its private counterpart is RSA_PKCS1_PRIVATE below. const RSA_PUBLIC: &str = "-----BEGIN PUBLIC KEY-----\n\ MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAx3J0fQcHp2ZlMI4rCVsY\n\ diff --git a/backend/windmill-common/src/instance_config.rs b/backend/windmill-common/src/instance_config.rs index b1ddb749a0..f4884d736e 100644 --- a/backend/windmill-common/src/instance_config.rs +++ b/backend/windmill-common/src/instance_config.rs @@ -831,6 +831,8 @@ pub struct CustomInstanceDbLogs { pub replication_user: String, #[serde(skip_serializing_if = "Option::is_none")] pub replication_user_error: Option, + #[serde(skip_serializing_if = "String::is_empty")] + pub user_connect: String, } // --------------------------------------------------------------------------- @@ -1027,9 +1029,10 @@ const NESTED_SENSITIVE_FIELDS: &[(&str, &[&str])] = &[ ), ( "object_store_cache_config", - &["secret_key", "serviceAccountKey"], + &["secret_key", "serviceAccountKey", "accessKey"], ), ("custom_instance_pg_databases", &["user_pwd"]), + ("github_enterprise_app", &["private_key"]), ]; fn redact_json_value(value: &serde_json::Value) -> serde_json::Value { @@ -2645,6 +2648,26 @@ mod tests { assert!(formatted.contains("****")); } + #[test] + fn format_setting_value_redacts_nested_credentials() { + let val = serde_json::json!({ + "type": "Azure", + "accountName": "acct", + "containerName": "c", + "accessKey": "azure-storage-account-key-12345" + }); + let formatted = format_setting_value("object_store_cache_config", &val); + assert!(!formatted.contains("azure-storage-account-key-12345")); + assert!(formatted.contains("acct")); + + let val = serde_json::json!({ + "app_id": 1, + "private_key": "-----BEGIN RSA PRIVATE KEY-----\nMIIEsecretbody\n-----END RSA PRIVATE KEY-----" + }); + let formatted = format_setting_value("github_enterprise_app", &val); + assert!(!formatted.contains("MIIEsecretbody")); + } + #[test] #[should_panic(expected = "literal_value() called on unresolved secret ref")] fn string_or_secret_ref_literal_value_panics_on_ref() { diff --git a/backend/windmill-common/src/jobs.rs b/backend/windmill-common/src/jobs.rs index 6f3248f3fb..796d1c999a 100644 --- a/backend/windmill-common/src/jobs.rs +++ b/backend/windmill-common/src/jobs.rs @@ -17,7 +17,7 @@ use crate::{ get_latest_deployed_hash_for_path, get_latest_flow_version_info_for_path, scripts::{get_full_hub_script_by_path, ScriptHash, ScriptLang}, utils::{StripPath, HTTP_CLIENT}, - worker::{to_raw_value, CUSTOM_TAGS_PER_WORKSPACE, WINDMILL_DIR}, + worker::{custom_tag_matches, to_raw_value, CUSTOM_TAGS_PER_WORKSPACE, WINDMILL_DIR}, workspaces::workspace_with_fork_ancestors, FlowVersionInfo, ScriptHashInfo, Tag, }; @@ -337,33 +337,70 @@ lazy_static::lazy_static! { // `is_super_admin` is passed in (not derived from an email here) so callers can // make it job-token-aware: a job's WM_TOKEN must never count as superadmin // (GHSA-hfh4-cx4h-3fcr). See `is_super_admin_authed` at the request wrapper. +// +// Custom tags judge `resolved_tag`, the queue the job actually lands on, and `tag` only as the +// job's author wrote it: a dynamic tag names any queue its values do. Callers go through +// `windmill_queue::check_tag_available_for_push`, which resolves it the way `push` does. +// `resolved_tag` is `None` when the values are not known yet: only an entry spelled like `tag` +// admits it then. `tag_workspace` resolves to what the job's `$workspace` stands for. pub async fn check_tag_available_for_workspace_internal( db: &DB, w_id: &str, tag: &str, + resolved_tag: Option<&str>, + tag_workspace: impl Future, is_super_admin: bool, scope_tags: Option>, ) -> error::Result<()> { - let mut is_tag_in_scope_tags = None; - let mut is_tag_in_workspace_custom_tags = false; - - if let Some(scope_tags) = scope_tags.as_ref() { - is_tag_in_scope_tags = Some(scope_tags.contains(&tag)); - } + let is_tag_in_scope_tags = scope_tags.as_ref().map(|scope_tags| { + scope_tags.contains(&tag) || resolved_tag.is_some_and(|r| scope_tags.contains(&r)) + }); let custom_tags_per_w = CUSTOM_TAGS_PER_WORKSPACE.load(); - if custom_tags_per_w.global.contains(&tag.to_string()) { - is_tag_in_workspace_custom_tags = true; - } else if let Some(specific_tag) = custom_tags_per_w.specific.get(tag) { - // Only a fork-scoped tag can match through the lineage, so every other tag keeps the - // ancestor lookup off the push path entirely. - let chain = if specific_tag.is_fork_scoped() { - workspace_with_fork_ancestors(db, w_id).await? - } else { - vec![w_id.to_string()] - }; - is_tag_in_workspace_custom_tags = specific_tag.applies_to_workspace(&chain); - } + // Only an entry reading `$workspace` needs it, so every other tag keeps its lookup off the + // push path. + let tag_workspace = if resolved_tag.is_some() + && custom_tags_per_w + .global + .iter() + .chain(custom_tags_per_w.specific.keys()) + .any(|t| t.contains("$workspace")) + { + tag_workspace.await + } else { + w_id.to_string() + }; + // A job whose tag is written exactly as an entry resolves inside what that entry admits, which + // is the only way in for an entry `custom_tag_matches` cannot turn into a pattern. + let admits = |entry: &str| { + entry == tag || resolved_tag.is_some_and(|r| custom_tag_matches(entry, r, &tag_workspace)) + }; + let scoped = custom_tags_per_w + .specific + .iter() + .filter(|(name, _)| admits(name)) + .collect::>(); + // Only a fork-scoped tag can match through the lineage, so every other tag keeps the + // ancestor lookup off the push path entirely. + let chain = if scoped.iter().any(|(_, t)| t.is_fork_scoped()) { + workspace_with_fork_ancestors(db, w_id).await? + } else { + vec![w_id.to_string()] + }; + // A tag listed by its own name with a scope stays inside that scope: a pattern it fits does + // not open it to other workspaces. Only listing it by name without a scope does. + let lands_on = resolved_tag.unwrap_or(tag); + let confined_by = custom_tags_per_w + .specific + .get_key_value(lands_on) + .filter(|(name, t)| { + !name.contains('$') + && !t.applies_to_workspace(&chain) + && !custom_tags_per_w.global.iter().any(|e| e == lands_on) + }); + let is_tag_in_workspace_custom_tags = confined_by.is_none() + && (custom_tags_per_w.global.iter().any(|entry| admits(entry)) + || scoped.iter().any(|(_, t)| t.applies_to_workspace(&chain))); match is_tag_in_scope_tags { Some(true) | None => { @@ -375,6 +412,12 @@ pub async fn check_tag_available_for_workspace_internal( } if !is_super_admin { + let tag = match resolved_tag { + Some(resolved_tag) if resolved_tag != tag => { + format!("{tag} (resolved to {resolved_tag})") + } + _ => tag.to_string(), + }; if scope_tags.is_some() && is_tag_in_scope_tags.is_some() { return Err(Error::BadRequest(format!( "Tag {tag} is not available in your scope" @@ -383,9 +426,35 @@ pub async fn check_tag_available_for_workspace_internal( if *TAGS_ARE_SENSITIVE { return Err(Error::BadRequest(format!("{tag} is not available to you"))); + } else if let Some((name, confining)) = confined_by { + let overridden = custom_tags_per_w + .global + .iter() + .filter(|entry| admits(entry)) + .cloned() + .chain( + scoped + .iter() + .filter(|(_, t)| t.applies_to_workspace(&chain)) + .map(|(n, t)| t.authored(n)), + ) + .collect::>(); + let overridden = if overridden.is_empty() { + String::new() + } else { + format!( + ", which takes precedence over the custom tags it also fits: {}", + overridden.join(", ") + ) + }; + return Err(Error::BadRequest(format!( + "Tag {tag} is not available in workspace {w_id}: the custom tag {} restricts it \ + to other workspaces{overridden}", + confining.authored(name) + ))); } else { return Err(error::Error::BadRequest(format!( - "Only super admins are allowed to use tags that are not included in the allowed CUSTOM_TAGS: {:?}", + "Tag {tag} is not included in the allowed CUSTOM_TAGS, which only super admins can go beyond: {:?}", custom_tags_per_w ))); } diff --git a/backend/windmill-common/src/jwt.rs b/backend/windmill-common/src/jwt.rs index cc2f1102bf..76df13b518 100644 --- a/backend/windmill-common/src/jwt.rs +++ b/backend/windmill-common/src/jwt.rs @@ -3,7 +3,6 @@ use base64::{engine::general_purpose::URL_SAFE_NO_PAD, Engine}; use hmac::{Hmac, Mac}; use serde::{de::DeserializeOwned, Serialize}; use sha2::Sha256; -use std::collections::HashSet; lazy_static::lazy_static! { pub static ref JWT_SECRET: arc_swap::ArcSwap = arc_swap::ArcSwap::from_pointee("".to_string()); @@ -44,18 +43,8 @@ pub async fn decode_with_internal_secret(token: &str) -> er } pub fn decode_without_verify(token: &str) -> anyhow::Result { - // Create a validation that skips all checks - let mut validation = jsonwebtoken::Validation::default(); - validation.insecure_disable_signature_validation(); - validation.validate_exp = false; - validation.validate_nbf = false; - validation.required_spec_claims = HashSet::new(); - - // Use an empty key since we're not verifying - let key = jsonwebtoken::DecodingKey::from_secret(&[]); - - // Decode the token - let token_data = jsonwebtoken::decode::(token, &key, &validation)?; + // No signature, expiry, nbf or required-claim checks: the caller wants the raw claims. + let token_data = jsonwebtoken::dangerous::insecure_decode::(token)?; Ok(token_data.claims) } @@ -74,3 +63,36 @@ pub async fn generate_signature(header_and_payload: &str) -> anyhow::Result Option { + // `alg` is a `KeyAlgorithm` (signing and encryption algorithms alike); only the + // signing subset maps onto `Algorithm`. + jwk.common + .key_algorithm + .and_then(|alg| alg.to_string().parse::().ok()) +} + +#[cfg(test)] +mod tests { + use super::jwk_algorithm; + use jsonwebtoken::{jwk::Jwk, Algorithm}; + + #[test] + fn jwk_algorithm_reads_alg_when_present() { + let with: Jwk = + serde_json::from_value(serde_json::json!({"kty":"RSA","alg":"RS256","n":"aa","e":"AQAB"})) + .unwrap(); + assert_eq!(jwk_algorithm(&with), Some(Algorithm::RS256)); + let without: Jwk = + serde_json::from_value(serde_json::json!({"kty":"RSA","n":"aa","e":"AQAB"})).unwrap(); + assert_eq!(jwk_algorithm(&without), None); + let encryption: Jwk = serde_json::from_value( + serde_json::json!({"kty":"RSA","alg":"RSA-OAEP","n":"aa","e":"AQAB"}), + ) + .unwrap(); + assert_eq!(jwk_algorithm(&encryption), None); + } +} diff --git a/backend/windmill-common/src/lib.rs b/backend/windmill-common/src/lib.rs index 52d1a85aec..3e74d8e17f 100644 --- a/backend/windmill-common/src/lib.rs +++ b/backend/windmill-common/src/lib.rs @@ -1024,6 +1024,20 @@ impl Future for TokioPgConnection { } } +impl TokioPgConnection { + /// Drive the connection and hand back what the server sends outside of a query's response — + /// notices above all, which driving it as a future silently discards. + pub fn poll_message( + &mut self, + cx: &mut core::task::Context<'_>, + ) -> core::task::Poll>> { + match self { + TokioPgConnection::Tls(conn) => conn.poll_message(cx), + TokioPgConnection::NoTls(conn) => conn.poll_message(cx), + } + } +} + impl PgDatabase { /// The role the connection logs in as, whichever way it authenticates. pub fn login_name(&self) -> &str { @@ -1460,8 +1474,163 @@ pub fn validate_dbname(dbname: &str) -> error::Result<()> { Ok(()) } +/// Lock the instance databases among `names` until `tx` ends, in a stable order. Taken by every +/// settings save naming an instance database, and by [`drop_unused_instance_database`]: a save +/// cannot start using a database between that drop's check that nothing does and the drop. +pub async fn lock_instance_databases<'a>( + tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, + names: impl IntoIterator, +) -> error::Result<()> { + let names: std::collections::BTreeSet<&str> = names.into_iter().collect(); + for name in names { + sqlx::query("SELECT pg_advisory_xact_lock(hashtext('instance_database:' || $1))") + .bind(name) + .execute(&mut **tx) + .await?; + } + Ok(()) +} + +/// Drop instance database `dbname`, which a request just created, unless a workspace names it as a +/// data table or a ducklake catalog — then it is kept, and who keeps it is what comes back. A +/// database is registered on the instance as soon as it is created, so a superadmin can point a +/// workspace at it before the request that created it gives up on it. +/// +/// Authorization: none. Callers MUST pass only a database the same request created and has not +/// handed to anything yet. +/// +/// The lock, the check and the drop share one connection: a second one taken from the pool while +/// the first is held could wait forever on a small pool. That connection is closed rather than +/// returned, since a session lock outlives the future holding it: a cancellation between taking +/// the lock and releasing it would otherwise hand a locked session back to the pool, where every +/// later settings save waits on it. Closing on drop covers the cancellation and still counts the +/// connection against the pool, which detaching it would not. +pub async fn drop_unused_instance_database(db: &DB, dbname: &str) -> error::Result { + let mut conn = db.acquire().await?; + conn.close_on_drop(); + let key = format!("instance_database:{dbname}"); + // A save blocked on this lock is about to name the database, but it may also roll back — a + // later validation of its own, a superadmin check, a cancelled request. So it is let through + // and what it committed is read, rather than taken as a user: trusting it would strand the + // database, whose name then blocks every retry. A fresh waiter can always arrive, so after a + // few rounds the database is kept instead; the name is then a superadmin's to drop from + // instance settings, since a retry fails on the name before reaching this cleanup. + let mut rounds = 0; + let dropped = loop { + if let Err(e) = sqlx::query("SELECT pg_advisory_lock(hashtext($1))") + .bind(&key) + .execute(&mut *conn) + .await + { + break Err(e.into()); + } + match drop_if_unused_on(&mut conn, dbname).await { + Ok(Cleanup::Waiter(_)) if rounds < 2 => { + rounds += 1; + // Releasing hands the lock to the waiter; re-taking it above then waits for that + // transaction to end, so the next round reads what it actually committed. + if let Err(e) = unlock_instance_database(&mut conn, &key).await { + break Err(e); + } + } + Ok(outcome) => break Ok(outcome), + Err(e) => break Err(e), + } + }; + // Dropping closes the connection, and the server releases the lock with the session, so + // nothing here depends on an unlock landing. + dropped +} + +async fn unlock_instance_database(conn: &mut sqlx::PgConnection, key: &str) -> error::Result<()> { + sqlx::query("SELECT pg_advisory_unlock(hashtext($1))") + .bind(key) + .execute(&mut *conn) + .await?; + Ok(()) +} + +/// What [`drop_unused_instance_database`] did, and for whom when it kept the database. +pub enum Cleanup { + Dropped, + /// Workspaces that name the database, as [`instance_database_users`] reports them. + InUse(Vec), + /// A transaction still blocked on the lock after the rounds above, so what it will commit + /// stays unknown, named as its `pid`. + Waiter(i32), +} + +async fn drop_if_unused_on(conn: &mut sqlx::PgConnection, dbname: &str) -> error::Result { + let users = instance_database_users(conn, dbname).await?; + if !users.is_empty() { + return Ok(Cleanup::InUse(users)); + } + // A settings save naming this database takes the same lock, so one waiting on it would read + // its own users after the drop and commit a reference to nothing. + if let Some(waiter) = waiting_for_instance_database(conn, dbname).await? { + return Ok(Cleanup::Waiter(waiter)); + } + drop_custom_instance_database_on(conn, dbname).await?; + Ok(Cleanup::Dropped) +} + +/// The `pid` of a transaction blocked on `dbname`'s instance-database lock. The lock is +/// taken by key, so `pg_locks` reports it split across `classid` and `objid`, and it lists every +/// database on the cluster — another Windmill on the same one holds its own locks under the same +/// key. +async fn waiting_for_instance_database( + conn: &mut sqlx::PgConnection, + dbname: &str, +) -> error::Result> { + let pid: Option = sqlx::query_scalar( + "SELECT l.pid FROM pg_locks l + WHERE l.locktype = 'advisory' AND NOT l.granted + AND l.database = (SELECT oid FROM pg_database WHERE datname = current_database()) + AND l.classid = ((hashtext('instance_database:' || $1)::bigint >> 32) & 4294967295)::oid + AND l.objid = (hashtext('instance_database:' || $1)::bigint & 4294967295)::oid + LIMIT 1", + ) + .bind(dbname) + .fetch_optional(&mut *conn) + .await?; + Ok(pid) +} + +/// The workspaces naming instance database `dbname` as a data table or a ducklake catalog. Taken +/// under [`lock_instance_databases`] for `dbname`, the answer holds until that lock is released. +/// +/// Authorization: none, and it reads every workspace's settings. Callers MUST pass only a database +/// their own request created, and may name the workspaces returned only to a caller allowed to +/// create or drop instance databases. +pub async fn instance_database_users( + conn: &mut sqlx::PgConnection, + dbname: &str, +) -> error::Result> { + Ok(sqlx::query_scalar( + "SELECT DISTINCT ws.workspace_id FROM workspace_settings ws + WHERE EXISTS (SELECT 1 FROM jsonb_each(CASE WHEN jsonb_typeof(ws.datatable->'datatables') = 'object' + THEN ws.datatable->'datatables' ELSE '{}'::jsonb END) dt + WHERE dt.value->'database'->>'resource_type' = 'instance' + AND dt.value->'database'->>'resource_path' = $1) + OR EXISTS (SELECT 1 FROM jsonb_each(CASE WHEN jsonb_typeof(ws.ducklake->'ducklakes') = 'object' + THEN ws.ducklake->'ducklakes' ELSE '{}'::jsonb END) dl + WHERE dl.value->'catalog'->>'resource_type' = 'instance' + AND dl.value->'catalog'->>'resource_path' = $1)", + ) + .bind(dbname) + .fetch_all(conn) + .await?) +} + /// Drop a custom instance database: validate, terminate connections, DROP DATABASE, remove from global_settings. pub async fn drop_custom_instance_database(db: &DB, dbname: &str) -> error::Result<()> { + drop_custom_instance_database_on(&mut *db.acquire().await?, dbname).await +} + +async fn drop_custom_instance_database_on( + conn: &mut sqlx::PgConnection, + dbname: &str, +) -> error::Result<()> { let dbname = dbname.trim(); validate_dbname(dbname)?; @@ -1476,7 +1645,7 @@ pub async fn drop_custom_instance_database(db: &DB, dbname: &str) -> error::Resu "SELECT EXISTS (SELECT 1 FROM pg_catalog.pg_database WHERE datname = $1)", dbname ) - .fetch_one(db) + .fetch_one(&mut *conn) .await? .unwrap_or(false); @@ -1487,7 +1656,7 @@ pub async fn drop_custom_instance_database(db: &DB, dbname: &str) -> error::Resu "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname = '{}' AND pid <> pg_backend_pid()", dbname.replace('\'', "''") )) - .execute(db) + .execute(&mut *conn) .await { tracing::warn!("Failed to terminate connections to '{}': {}", dbname, e); @@ -1496,7 +1665,7 @@ pub async fn drop_custom_instance_database(db: &DB, dbname: &str) -> error::Resu // Drop the database // SAFETY: `dbname` has been validated via validate_dbname() before reaching this point. sqlx::query(&format!("DROP DATABASE IF EXISTS \"{}\"", dbname)) - .execute(db) + .execute(&mut *conn) .await .map_err(|e| { error::Error::internal_err(format!("Failed to drop database '{}': {}", dbname, e)) @@ -1512,7 +1681,7 @@ pub async fn drop_custom_instance_database(db: &DB, dbname: &str) -> error::Resu r#"UPDATE global_settings SET value = value #- ARRAY['databases', $1] WHERE name = 'custom_instance_pg_databases'"#, dbname ) - .execute(db) + .execute(&mut *conn) .await?; Ok(()) @@ -1586,6 +1755,39 @@ pub async fn create_custom_instance_database( error::Error::internal_err(format!("Failed to create database '{}': {}", dbname, e)) })?; + // Nothing names a database that failed past this point, and its name blocks the retry: drop it + // rather than leave it behind. + if let Err(e) = finish_custom_instance_database(db, dbname, tag).await { + match drop_unused_instance_database(db, dbname).await { + Ok(Cleanup::InUse(users)) => tracing::warn!( + "Kept '{dbname}' after failing to set it up: workspaces {} use it", + users.join(", ") + ), + Ok(Cleanup::Waiter(pid)) => tracing::warn!( + "Kept '{dbname}' after failing to set it up: a request (pid {pid}) is still \ + waiting to name it. Drop it from instance settings once it is unused." + ), + Ok(Cleanup::Dropped) => {} + Err(drop_err) => { + tracing::error!("Could not drop '{dbname}' after failing to set it up: {drop_err}") + } + } + return Err(e); + } + + // A data table role can only reach a database it may CONNECT to, and PUBLIC's default CONNECT + // would otherwise let every role in regardless of what this instance defines. Best-effort: a + // failure here leaves the database usable as `admin`, and the next role change repairs it. + if let Err(e) = crate::datatable_roles::converge_connect_grants(db, dbname).await { + tracing::warn!("Could not set CONNECT grants on instance database '{dbname}': {e}"); + } + + tracing::info!("Created custom instance database '{}'", dbname); + Ok(()) +} + +/// Grant `custom_instance_user` its privileges on a database just created, and register it. +async fn finish_custom_instance_database(db: &DB, dbname: &str, tag: &str) -> error::Result<()> { // Grant permissions to custom_instance_user let wmill_pg_creds = PgDatabase::parse_uri(&get_database_url().await?.as_str().await)?; let new_pg_creds = PgDatabase { dbname: dbname.to_string(), ..wmill_pg_creds }; @@ -1620,15 +1822,6 @@ pub async fn create_custom_instance_database( ) .execute(db) .await?; - - // A data table role can only reach a database it may CONNECT to, and PUBLIC's default CONNECT - // would otherwise let every role in regardless of what this instance defines. Best-effort: a - // failure here leaves the database usable as `admin`, and the next role change repairs it. - if let Err(e) = crate::datatable_roles::converge_connect_grants(db, dbname).await { - tracing::warn!("Could not set CONNECT grants on instance database '{dbname}': {e}"); - } - - tracing::info!("Created custom instance database '{}'", dbname); Ok(()) } diff --git a/backend/windmill-common/src/query_builders.rs b/backend/windmill-common/src/query_builders.rs index 60d74f189d..546d2d454b 100644 --- a/backend/windmill-common/src/query_builders.rs +++ b/backend/windmill-common/src/query_builders.rs @@ -329,9 +329,11 @@ pub fn try_expand_internal_db_query( "ALTER_TABLE" => expand_alter_table(json_str, db_type).map(ExpandedQuery::sql), "CREATE_SCHEMA" => expand_create_schema(json_str, db_type).map(ExpandedQuery::sql), "DROP_SCHEMA" => expand_drop_schema(json_str, db_type).map(ExpandedQuery::sql), + "RENAME_SCHEMA" => expand_rename_schema(json_str, db_type).map(ExpandedQuery::sql), // Metadata queries "LOAD_TABLE_METADATA" => expand_load_table_metadata(json_str, db_type), "FOREIGN_KEYS" => expand_foreign_keys(json_str, db_type).map(ExpandedQuery::sql), + "ALL_FOREIGN_KEYS" => expand_all_foreign_keys(db_type).map(ExpandedQuery::sql), "PRIMARY_KEY_CONSTRAINT" => { expand_primary_key_constraint(json_str, db_type).map(ExpandedQuery::sql) } @@ -1716,6 +1718,13 @@ struct DropSchemaPayload { ducklake: Option, } +#[derive(Deserialize)] +struct RenameSchemaPayload { + schema: String, + new_schema: String, + ducklake: Option, +} + #[derive(Debug, Clone, Deserialize)] struct TableEditorColumn { name: String, @@ -2004,6 +2013,23 @@ fn expand_drop_schema(json_str: &str, db_type: DbType) -> Result Ok(maybe_wrap_ducklake(query, p.ducklake.as_deref())) } +fn expand_rename_schema(json_str: &str, db_type: DbType) -> Result { + let p: RenameSchemaPayload = serde_json::from_str(json_str) + .map_err(|e| format!("Invalid RENAME_SCHEMA payload: {}", e))?; + if !matches!(db_type, DbType::Postgresql | DbType::Snowflake) || p.ducklake.is_some() { + return Err(format!( + "Renaming a schema is not supported on {:?}", + db_type + )); + } + let query = format!( + "ALTER SCHEMA {} RENAME TO {};", + qi(&p.schema, db_type), + qi(&p.new_schema, db_type) + ); + Ok(query) +} + fn expand_create_table(json_str: &str, db_type: DbType) -> Result { let p: CreateTablePayload = serde_json::from_str(json_str) .map_err(|e| format!("Invalid CREATE_TABLE payload: {}", e))?; @@ -2598,7 +2624,9 @@ WHERE table_catalog = current_database()", ) } else { ( - "\nWHERE c.relkind = 'r' AND a.attnum > 0 AND NOT a.attisdropped\n AND ns.nspname != 'pg_catalog' AND ns.nspname != 'information_schema'".to_string(), + // pg_catalog is readable by everyone: without the privilege check this lists + // tables of schemas the connection's role cannot even enter. + "\nWHERE c.relkind = 'r' AND a.attnum > 0 AND NOT a.attisdropped\n AND ns.nspname != 'pg_catalog' AND ns.nspname != 'information_schema'\n AND has_schema_privilege(ns.oid, 'USAGE')".to_string(), ",\n ns.nspname AS schema_name,\n c.relname AS table_name".to_string(), "\nJOIN pg_catalog.pg_class c ON a.attrelid = c.oid\nJOIN pg_catalog.pg_namespace ns ON c.relnamespace = ns.oid".to_string(), "ns.nspname, c.relname, a.attnum".to_string(), @@ -2710,6 +2738,47 @@ fn expand_foreign_keys(json_str: &str, db_type: DbType) -> Result Result { + make_all_foreign_keys_query(db_type) +} + +/// Every foreign key of the database, one row per referencing column. +/// +/// Read from `pg_constraint` rather than `information_schema`: the latter's +/// `constraint_column_usage` loses the pairing of a composite key's columns, +/// which the diagram needs to line a source column up with the column it points +/// at. `unnest(conkey, confkey)` keeps them zipped in declaration order. +fn make_all_foreign_keys_query(db_type: DbType) -> Result { + if db_type != DbType::Postgresql { + return Err(format!( + "The schema diagram is only supported for PostgreSQL, not {:?}", + db_type + )); + } + Ok(String::from( + "SELECT + con.conname as fk_constraint_name, + src_ns.nspname as source_schema, + src.relname as source_table, + src_att.attname as source_column, + tgt_ns.nspname as target_schema, + tgt.relname as target_table, + tgt_att.attname as target_column, + k.ord as ordinal +FROM pg_catalog.pg_constraint con + JOIN pg_catalog.pg_class src ON src.oid = con.conrelid + JOIN pg_catalog.pg_namespace src_ns ON src_ns.oid = src.relnamespace + JOIN pg_catalog.pg_class tgt ON tgt.oid = con.confrelid + JOIN pg_catalog.pg_namespace tgt_ns ON tgt_ns.oid = tgt.relnamespace + JOIN LATERAL unnest(con.conkey, con.confkey) WITH ORDINALITY AS k(src_attnum, tgt_attnum, ord) ON true + JOIN pg_catalog.pg_attribute src_att ON src_att.attrelid = src.oid AND src_att.attnum = k.src_attnum + JOIN pg_catalog.pg_attribute tgt_att ON tgt_att.attrelid = tgt.oid AND tgt_att.attnum = k.tgt_attnum +WHERE con.contype = 'f' + AND src_ns.nspname NOT IN ('pg_catalog', 'information_schema') +ORDER BY src_ns.nspname, src.relname, con.conname, k.ord;", + )) +} + fn expand_primary_key_constraint(json_str: &str, db_type: DbType) -> Result { let p: PrimaryKeyConstraintPayload = serde_json::from_str(json_str) .map_err(|e| format!("Invalid PRIMARY_KEY_CONSTRAINT payload: {}", e))?; @@ -4101,6 +4170,13 @@ mod tests { assert_eq!(sql, "DROP SCHEMA \"old_schema\" CASCADE;"); } + #[test] + fn test_expand_rename_schema() { + let marker = r#"-- WM_INTERNAL_DB_RENAME_SCHEMA {"schema":"old","new_schema":"new"}"#; + let sql = expand_code(marker, &ScriptLang::Postgresql); + assert_eq!(sql, "ALTER SCHEMA \"old\" RENAME TO \"new\";"); + } + #[test] fn test_expand_create_schema_with_ducklake() { let marker = r#"-- WM_INTERNAL_DB_CREATE_SCHEMA {"schema":"s","ducklake":"lake"}"#; @@ -4468,6 +4544,7 @@ mod tests { assert!(sql.contains("schema_name")); assert!(sql.contains("table_name")); assert!(sql.contains("c.relkind = 'r'")); + assert!(sql.contains("has_schema_privilege(ns.oid, 'USAGE')")); } #[test] @@ -4686,6 +4763,26 @@ mod tests { assert!(sql.starts_with("ATTACH 'ducklake://lake' AS dl;USE dl;\n")); } + // ----------------------------------------------------------------------- + // ALL_FOREIGN_KEYS + // ----------------------------------------------------------------------- + + #[test] + fn test_expand_all_foreign_keys_postgresql() { + let marker = r#"-- WM_INTERNAL_DB_ALL_FOREIGN_KEYS {}"#; + let sql = expand_code(marker, &ScriptLang::Postgresql); + assert!(sql.contains("con.contype = 'f'")); + // Columns of a composite key must stay zipped with the ones they point at. + assert!(sql.contains("unnest(con.conkey, con.confkey) WITH ORDINALITY")); + } + + #[test] + fn test_expand_all_foreign_keys_rejects_non_postgres() { + let marker = r#"-- WM_INTERNAL_DB_ALL_FOREIGN_KEYS {}"#; + let result = try_expand_internal_db_query(marker, &ScriptLang::Mysql); + assert!(result.unwrap().is_err()); + } + // ----------------------------------------------------------------------- // PRIMARY_KEY_CONSTRAINT // ----------------------------------------------------------------------- diff --git a/backend/windmill-common/src/ssrf.rs b/backend/windmill-common/src/ssrf.rs index 31dee749e6..0cdf7ffa4f 100644 --- a/backend/windmill-common/src/ssrf.rs +++ b/backend/windmill-common/src/ssrf.rs @@ -8,6 +8,8 @@ pub const ALLOW_PRIVATE_SAML_METADATA_URLS_ENV: &str = "ALLOW_PRIVATE_SAML_METAD pub const ALLOW_PRIVATE_GUEST_JWKS_URLS_ENV: &str = "ALLOW_PRIVATE_GUEST_JWKS_URLS"; +pub const ALLOW_PRIVATE_WEBHOOK_URLS_ENV: &str = "ALLOW_PRIVATE_WEBHOOK_URLS"; + /// Lets every git call reach hosts on a private network, whoever it is made for. /// Without it, [`private_git_host_allowed`] decides. pub const ALLOW_LOCAL_GIT_REMOTES_ENV: &str = "ALLOW_LOCAL_GIT_REMOTES"; @@ -207,6 +209,12 @@ pub fn allow_private_saml_metadata_urls() -> bool { .is_some_and(|v| v == "true" || v == "1") } +fn allow_private_webhook_urls() -> bool { + std::env::var(ALLOW_PRIVATE_WEBHOOK_URLS_ENV) + .ok() + .is_some_and(|v| v == "true" || v == "1") +} + fn allow_local_git_remotes() -> bool { std::env::var(ALLOW_LOCAL_GIT_REMOTES_ENV) .ok() @@ -321,6 +329,26 @@ pub async fn validate_mcp_server_url(url: &str) -> Result Result { + let parsed = + url::Url::parse(url).map_err(|e| SsrfValidationError::InvalidUrl(e.to_string()))?; + + match parsed.scheme() { + "http" | "https" => {} + scheme => return Err(SsrfValidationError::DisallowedScheme(scheme.to_string())), + } + + let host = parsed.host_str().ok_or(SsrfValidationError::MissingHost)?; + + if allow_private_webhook_urls() { + return Ok(ValidatedTarget::unpinned(host)); + } + + validate_url_for_ssrf(url).await +} + /// Validate an MCP-related URL and return the [`ValidatedTarget`] so the caller /// can pin the connect: the OAuth registration/discovery/token requests carry /// secrets, so they must target the validated address (see @@ -358,6 +386,16 @@ pub fn saml_ssrf_error_message(e: &SsrfValidationError) -> String { } } +pub fn webhook_ssrf_error_message(e: &SsrfValidationError) -> String { + match e { + SsrfValidationError::Private { .. } => format!( + "{e}. If you need to use private/internal webhook URLs, \ + set the {ALLOW_PRIVATE_WEBHOOK_URLS_ENV}=true environment variable" + ), + _ => e.to_string(), + } +} + fn is_private_ip(ip: &IpAddr) -> bool { match ip { IpAddr::V4(ipv4) => is_private_ipv4(ipv4), @@ -684,6 +722,71 @@ mod tests { )); } + struct PrivateWebhookUrlsEnvGuard { + previous: Option, + } + + impl PrivateWebhookUrlsEnvGuard { + fn set(value: Option<&str>) -> Self { + let previous = std::env::var(ALLOW_PRIVATE_WEBHOOK_URLS_ENV).ok(); + match value { + Some(value) => std::env::set_var(ALLOW_PRIVATE_WEBHOOK_URLS_ENV, value), + None => std::env::remove_var(ALLOW_PRIVATE_WEBHOOK_URLS_ENV), + } + Self { previous } + } + } + + impl Drop for PrivateWebhookUrlsEnvGuard { + fn drop(&mut self) { + match &self.previous { + Some(value) => std::env::set_var(ALLOW_PRIVATE_WEBHOOK_URLS_ENV, value), + None => std::env::remove_var(ALLOW_PRIVATE_WEBHOOK_URLS_ENV), + } + } + } + + #[tokio::test] + async fn validate_webhook_url_blocks_private_by_default_with_env_hint() { + let _lock = TEST_ENV_LOCK.lock().await; + let _guard = PrivateWebhookUrlsEnvGuard::set(None); + + let private_error = validate_webhook_url("http://127.0.0.1/hook") + .await + .unwrap_err(); + assert!(matches!( + private_error, + SsrfValidationError::Private { resolved: false } + )); + assert!( + webhook_ssrf_error_message(&private_error).contains("ALLOW_PRIVATE_WEBHOOK_URLS=true") + ); + + let invalid_error = validate_webhook_url("ftp://example.com/hook") + .await + .unwrap_err(); + assert!( + !webhook_ssrf_error_message(&invalid_error).contains(ALLOW_PRIVATE_WEBHOOK_URLS_ENV) + ); + } + + #[tokio::test] + async fn validate_webhook_url_allows_private_when_env_is_set_but_keeps_syntax_guards() { + let _lock = TEST_ENV_LOCK.lock().await; + let _guard = PrivateWebhookUrlsEnvGuard::set(Some("true")); + + assert!(validate_webhook_url("http://127.0.0.1/hook").await.is_ok()); + assert!(validate_webhook_url("http://10.0.0.1/hook").await.is_ok()); + assert!(matches!( + validate_webhook_url("file:///etc/passwd").await, + Err(SsrfValidationError::DisallowedScheme(_)) + )); + assert!(matches!( + validate_webhook_url("not-a-url").await, + Err(SsrfValidationError::InvalidUrl(_)) + )); + } + #[test] fn private_git_hosts_are_refused_to_non_admins_and_on_cloud() { use GitRemoteCaller::{AdminOrSystem, NonAdmin}; diff --git a/backend/windmill-common/src/user_drafts.rs b/backend/windmill-common/src/user_drafts.rs index aed9b5c5de..ebd0ce363f 100644 --- a/backend/windmill-common/src/user_drafts.rs +++ b/backend/windmill-common/src/user_drafts.rs @@ -160,6 +160,34 @@ impl UserDraftItemKind { } } + /// The `draft.value` key holding the user-typed target path — where a + /// deploy of this draft would land when the user staged a rename. A script + /// draft round-trips its own `path`; flows and apps write a separate + /// `draft_path`, and only when it differs from the row's path. + /// + /// `None` for every kind with no editor to stage a rename in — a resource, + /// a variable, a trigger. Their drafts carry no such key, so `Some` is also + /// the test for whether a kind can be moved at all. + pub fn typed_path_field(&self) -> Option<&'static str> { + use UserDraftItemKind::*; + match self { + Script => Some("path"), + Flow | App | RawApp => Some("draft_path"), + _ => None, + } + } + + /// The other of the two path keys a draft value can carry. The editors mirror + /// the typed path into it while it differs from the row's path, so a move that + /// rewrote only `typed_path_field` would leave the mirror naming the old + /// location — and the loaders prefer the mirror, which un-does the move. + pub fn mirror_path_field(&self) -> Option<&'static str> { + match self.typed_path_field()? { + "path" => Some("draft_path"), + _ => Some("path"), + } + } + /// Whether OTHER users' drafts at a path are visible to a viewer (the /// "others are editing" list, owner circles, and the `get_draft_for_user` /// View JSON / Fork endpoint). Enabled only for the full-page editor items @@ -212,6 +240,12 @@ pub struct WithDraftOverlay { pub is_draft: bool, #[serde(skip_serializing_if = "Option::is_none")] pub draft_saved_at: Option>, + /// The deployed version the draft forked from (`draft.base`), as text + /// whatever the kind. The editor compares it to the head it loaded to tell + /// a draft that is behind. Absent when there is no draft or it was never + /// forked from a deploy. + #[serde(skip_serializing_if = "Option::is_none")] + pub draft_base: Option, /// True when no deployed row exists at this path: `inner` is only a /// best-effort stand-in synthesized from the draft and only `draft` is /// canonical. Frontend uses this to disable "diff/reset vs deployed" and @@ -297,6 +331,7 @@ where inner: Box::new(deployed), is_draft: false, draft_saved_at: None, + draft_base: None, no_deployed: false, draft: None, other_drafts_users: Vec::new(), @@ -317,7 +352,7 @@ where // row when an owned one exists. let row = sqlx::query!( r#"SELECT value as "value!: sqlx::types::Json>", - created_at + created_at, base FROM draft WHERE workspace_id = $1 AND (email = $2 OR email IS NULL) @@ -338,6 +373,7 @@ where inner: Box::new(deployed), is_draft: false, draft_saved_at: None, + draft_base: None, no_deployed: false, draft: None, other_drafts_users, @@ -350,6 +386,7 @@ where inner: Box::new(deployed), is_draft: true, draft_saved_at: Some(row.created_at), + draft_base: row.base, no_deployed: false, draft: Some(draft_json), other_drafts_users, @@ -468,7 +505,8 @@ pub async fn overlay_or_draft_only( } } -/// Delete the drafts an address owns, across every workspace. +/// Delete the drafts an address owns, and the move records routing saves to them, +/// across every workspace. /// /// `draft.email` carries no foreign key to `password`: a draft's owner is any principal the /// instance authenticates, and an external JWT's subject never has a `password` row. Deleting an @@ -479,12 +517,14 @@ pub async fn overlay_or_draft_only( /// No authorization of its own: it acts instance-wide on whatever address it is handed, so the /// caller must already have authorized removing that account (superadmin, the account's own /// holder, or SCIM). -pub async fn delete_drafts_of_email<'c>( - executor: impl sqlx::PgExecutor<'c>, - email: &str, -) -> Result<()> { +pub async fn delete_drafts_of_email(conn: &mut sqlx::PgConnection, email: &str) -> Result<()> { sqlx::query!("DELETE FROM draft WHERE email = $1", email) - .execute(executor) + .execute(&mut *conn) + .await?; + // The records routing saves to those drafts go with them: one left behind would send + // a later holder of the address to a draft that no longer exists. + sqlx::query!("DELETE FROM draft_move WHERE email = $1", email) + .execute(&mut *conn) .await?; Ok(()) } @@ -526,6 +566,29 @@ pub async fn rename_drafts_of_email( ) .execute(&mut *conn) .await?; + // The records that route saves to those drafts follow the same address, or a save + // still addressed to a path the account moved away from would start a second draft + // there. The moving account's record for a path wins, as its draft just did. + sqlx::query!( + "DELETE FROM draft_move dest + WHERE dest.email = $1 + AND EXISTS (SELECT 1 FROM draft_move src + WHERE src.email = $2 + AND src.workspace_id = dest.workspace_id + AND src.old_path = dest.old_path + AND src.typ = dest.typ)", + new_email, + old_email + ) + .execute(&mut *conn) + .await?; + sqlx::query!( + "UPDATE draft_move SET email = $1 WHERE email = $2", + new_email, + old_email + ) + .execute(&mut *conn) + .await?; Ok(()) } @@ -582,6 +645,247 @@ pub async fn delete_own_draft_for_path( Ok(()) } +/// Carry every draft at `old_path` over to `new_path` when an item MOVES +/// (rename or relocation). A draft is bound to its item by nothing but the path +/// string, so without this a move detaches every draft on the item. No owner +/// filter: teammates' rows and the legacy NULL-email row follow too. +/// +/// **The caller must have authorized the underlying item move first.** This rewrites +/// rows the caller has no permission on and enforces nothing itself, taking both paths +/// on trust: it is safe only inside a deploy that has already cleared them, and is a +/// cross-user write with no gate anywhere else. +/// +/// The value keeps its base version, so every carried draft reads as behind the head the +/// move minted. Of its two path keys, one still naming `old_path` follows the row (the +/// editors write the item's own path there on every save, so it is the item's path and +/// not a staged rename); any other value is a rename the user staged, and is kept. +/// +/// A draft already at `new_path` occupies it the way a deployed item does, so the move is +/// refused with `BadRequest` inside the deploy's transaction, refusing the rename itself: +/// moving onto it would merge two items or strand the row that lost. A row a `draft_move` +/// record already brought there from `old_path` is this rename's own draft and does not +/// count: deploying it would otherwise be refused against itself, with no way out. +pub async fn move_drafts_for_path( + tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, + w_id: &str, + kinds: &[UserDraftItemKind], + old_path: &str, + new_path: &str, +) -> Result<()> { + let typs = kinds.iter().map(|k| k.as_str()).collect::>(); + // Named by workspace username, as the editors name other users' drafts: the + // caller is often not the owner, and cannot clear a draft they cannot find. + let owners = sqlx::query!( + r#"SELECT d.email IS NULL as "legacy!", COALESCE(u.username, p.username) as username + FROM draft d + LEFT JOIN usr u ON u.workspace_id = d.workspace_id AND u.email = d.email + LEFT JOIN password p ON p.email = d.email AND p.super_admin = true + WHERE d.workspace_id = $1 AND d.path = $2 AND d.typ::text = ANY($3::text[]) + -- A row a move already brought here from the very path being renamed is this + -- rename's own draft, not an item in its way: the record says which owners + -- it moved (all of them for an item move, one for a draft-only move). Any + -- other row is a second item and still collides. + AND NOT EXISTS ( + SELECT 1 FROM draft_move m + WHERE m.workspace_id = $1 AND m.typ::text = ANY($3::text[]) + AND m.old_path = $4 AND m.new_path = $2 + AND (m.email IS NULL OR m.email IS NOT DISTINCT FROM d.email) + ) + ORDER BY 2"#, + w_id, + new_path, + &typs as &[&str], + old_path, + ) + .fetch_all(&mut **tx) + .await?; + if !owners.is_empty() { + let names = owners + .into_iter() + .map(|o| match (o.username, o.legacy) { + (Some(name), _) => name, + (None, true) => "a legacy workspace draft".to_string(), + (None, false) => "another user".to_string(), + }) + .collect::>() + .join(", "); + return Err(crate::error::Error::BadRequest(format!( + "'{new_path}' already has a draft on it ({names}) — it must be moved or discarded first" + ))); + } + // `draft.value` is `json`, so a NUL escape left by a pre-sanitizer write makes + // `to_jsonb` raise 22P05. `clean` is `strip_json_nul`'s parity rule in SQL (escaped + // backslashes parked on chr(1), which a `json` value's text cannot hold, so nothing + // collides with it; chr(92) spells the backslash so no escape sequence reaches this + // file). A row whose keys need re-pointing is therefore rewritten clean rather than + // left naming the old path, and one that needs nothing keeps its value byte for byte. + sqlx::query!( + r#"UPDATE draft AS d + SET path = $3::text, + value = ( + SELECT CASE + WHEN s.clean -> 'path' = to_jsonb($2::text) + OR s.clean -> 'draft_path' = to_jsonb($2::text) + THEN to_json( + s.clean + || CASE WHEN s.clean -> 'path' = to_jsonb($2::text) + THEN jsonb_build_object('path', $3::text) + ELSE '{}'::jsonb END + || CASE WHEN s.clean -> 'draft_path' = to_jsonb($2::text) + THEN jsonb_build_object('draft_path', $3::text) + ELSE '{}'::jsonb END + ) + ELSE d.value + END + FROM (SELECT replace(replace(replace(d.value::text, chr(92) || chr(92), chr(1)), + chr(92) || 'u0000', ''), chr(1), chr(92) || chr(92))::jsonb AS clean) s + ) + WHERE d.workspace_id = $1 + AND d.path = $2::text + AND d.typ::text = ANY($4::text[])"#, + w_id, + old_path, + new_path, + &typs as &[&str], + ) + .execute(&mut **tx) + .await?; + record_draft_move(tx, w_id, kinds, old_path, new_path, None).await +} + +/// Record that the drafts at `old_path` now live at `new_path`, so a draft save still +/// addressed to `old_path` lands on them (see `update_draft`). `email` scopes the +/// record to one user's draft-only move; `None` is a deployed item's move, for everyone. +/// +/// Kept to one hop: records pointing at `old_path` are re-pointed (an owner's move +/// leaves the item's own record alone and copies it into its own scope instead), and +/// records leaving either path are replaced, since `new_path` now holds the item. +/// +/// **The caller must have authorized the move first.** A record routes every later +/// draft write at `old_path` (any owner's, for an item move), and enforces nothing +/// itself. +/// +/// A record outlives the editors that need it: it ends when a later move touches either +/// path or an item is deployed at `old_path`, so a save meaning to start a NEW draft at a +/// vacated path would be routed instead. Nothing does that today: every surface parks a +/// new item at a minted `u//draft_` key (`mintDraftPath.ts`). +pub async fn record_draft_move( + tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, + w_id: &str, + kinds: &[UserDraftItemKind], + old_path: &str, + new_path: &str, + email: Option<&str>, +) -> Result<()> { + let typs = kinds.iter().map(|k| k.as_str()).collect::>(); + sqlx::query!( + "DELETE FROM draft_move + WHERE workspace_id = $1 AND typ::text = ANY($2::text[]) + AND old_path IN ($3, $4) AND ($5::text IS NULL OR email = $5)", + w_id, + &typs as &[&str], + old_path, + new_path, + email, + ) + .execute(&mut **tx) + .await?; + // Routes that ended at the destination before this move describe drafts that were + // carried there for an item this one is replacing: left alive, a save addressed to + // the start of that chain would land on this move's draft instead. + sqlx::query!( + "DELETE FROM draft_move + WHERE workspace_id = $1 AND typ::text = ANY($2::text[]) + AND new_path = $4 AND old_path <> $3 AND ($5::text IS NULL OR email = $5)", + w_id, + &typs as &[&str], + old_path, + new_path, + email, + ) + .execute(&mut **tx) + .await?; + sqlx::query!( + "UPDATE draft_move SET new_path = $4 + WHERE workspace_id = $1 AND typ::text = ANY($2::text[]) + AND new_path = $3 AND ($5::text IS NULL OR email = $5)", + w_id, + &typs as &[&str], + old_path, + new_path, + email, + ) + .execute(&mut **tx) + .await?; + // An owner's move must not re-point what everyone else follows, so the item's own + // records ending at `old_path` are copied into this scope: a save addressed to the + // start of that chain still reaches this destination in one hop. Only those: another + // user's owner-scoped record routes their own draft-only item, which this move has + // nothing to do with. + if email.is_some() { + sqlx::query!( + "INSERT INTO draft_move (workspace_id, typ, old_path, new_path, email) + SELECT m.workspace_id, m.typ, m.old_path, $4, $5::text + FROM draft_move m + WHERE m.workspace_id = $1 AND m.typ::text = ANY($2::text[]) + AND m.new_path = $3 AND m.email IS NULL + AND NOT EXISTS ( + SELECT 1 FROM draft_move o + WHERE o.workspace_id = m.workspace_id AND o.typ = m.typ + AND o.old_path = m.old_path AND o.email = $5::text + )", + w_id, + &typs as &[&str], + old_path, + new_path, + email, + ) + .execute(&mut **tx) + .await?; + } + sqlx::query!( + "INSERT INTO draft_move (workspace_id, typ, old_path, new_path, email) + SELECT $1, t::draft_kind, $3, $4, $5 FROM unnest($2::text[]) t", + w_id, + &typs as &[&str], + old_path, + new_path, + email, + ) + .execute(&mut **tx) + .await?; + Ok(()) +} + +/// Drop the move records at `path`: an item was just created there, so saves addressed to +/// it are its own (records leaving `path`). A deploy that is not a rename also drops the +/// records arriving, which point at an item that no longer owns the path; `keep_from`, +/// the path a rename came from, suppresses that, since the chain ending here is the one +/// this very deploy just wrote. +/// +/// **The caller must have authorized the deploy that created the item first.** +/// Dropping a record sends later draft writes at `path` back to `path`. +pub async fn clear_draft_moves_from( + tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, + w_id: &str, + kinds: &[UserDraftItemKind], + path: &str, + keep_from: Option<&str>, +) -> Result<()> { + let typs = kinds.iter().map(|k| k.as_str()).collect::>(); + sqlx::query!( + "DELETE FROM draft_move WHERE workspace_id = $1 AND typ::text = ANY($2::text[]) + AND (old_path = $3 OR (new_path = $3 AND $4::text IS NULL))", + w_id, + &typs as &[&str], + path, + keep_from, + ) + .execute(&mut **tx) + .await?; + Ok(()) +} + /// Fetch the authed user's draft as a standalone payload, for "get by path" /// routes when no deployed row exists but a draft might. Returns it as a /// `WithDraftOverlay` with `inner` and `draft` both set to the same JSON and @@ -600,7 +904,7 @@ pub async fn fetch_draft_only( // Own draft first, legacy NULL-email row as fallback (see `maybe_overlay_draft`). let row = sqlx::query!( r#"SELECT value as "value!: sqlx::types::Json>", - created_at + created_at, base FROM draft WHERE workspace_id = $1 AND (email = $2 OR email IS NULL) @@ -631,6 +935,7 @@ pub async fn fetch_draft_only( inner: Box::new(draft_json.clone()), is_draft: true, draft_saved_at: Some(row.created_at), + draft_base: row.base, no_deployed: true, draft: Some(draft_json), other_drafts_users, diff --git a/backend/windmill-common/src/utils.rs b/backend/windmill-common/src/utils.rs index fd0bd634ea..29fc9e99f3 100644 --- a/backend/windmill-common/src/utils.rs +++ b/backend/windmill-common/src/utils.rs @@ -479,6 +479,28 @@ pub fn paginate(pagination: Pagination) -> (usize, usize) { (per_page, offset) } +/// [`paginate`] for a listing that answers whole unless a size is asked for: the deploy +/// histories, which the history panels and the CLI read unpaged while the diff picker takes +/// a page at a time. An asked-for size is still clamped, and the offset saturates rather +/// than wrapping, so no caller can turn this into an unbounded scan or a negative bind. +pub fn paginate_optional(pagination: Pagination) -> (i64, i64) { + // Naming neither parameter asks for the whole listing, the contract these endpoints + // have always answered on. Naming either makes it a page like any other listing's. + if pagination.page.is_none() && pagination.per_page.is_none() { + return (i64::MAX, 0); + } + let per_page = pagination + .per_page + .unwrap_or(DEFAULT_PER_PAGE) + .clamp(1, MAX_PER_PAGE) as i64; + // Bound before Postgres sees it: an unchecked cast of a caller-controlled page becomes + // a negative OFFSET, which is an error rather than an empty page. + let offset = i64::try_from(pagination.page.unwrap_or(1).max(1) - 1) + .unwrap_or(i64::MAX) + .saturating_mul(per_page); + (per_page, offset) +} + pub fn paginate_without_limits(pagination: Pagination) -> (usize, usize) { let per_page = pagination.per_page.unwrap_or(MAX_PER_PAGE); let offset = (pagination.page.unwrap_or(1).max(1) - 1) * per_page; @@ -1672,6 +1694,31 @@ pub fn truncate_with_ellipsis(s: &str, max_chars: usize) -> String { mod tests { use super::*; + #[test] + fn test_paginate_optional_answers_whole_but_bounds_what_is_asked_for() { + // Nothing asked for: every row there can be, which is what the history panels and + // the CLI read. + assert_eq!( + paginate_optional(Pagination { page: None, per_page: None }), + (i64::MAX, 0) + ); + assert_eq!( + paginate_optional(Pagination { page: Some(3), per_page: Some(20) }), + (20, 40) + ); + // An asked-for size is still capped, so no caller turns this into an unbounded scan. + assert_eq!( + paginate_optional(Pagination { page: None, per_page: Some(usize::MAX) }), + (MAX_PER_PAGE as i64, 0) + ); + // A page nobody could mean lands past the end rather than going negative, which + // Postgres would reject outright. + let (per_page, offset) = + paginate_optional(Pagination { page: Some(usize::MAX), per_page: Some(20) }); + assert_eq!(per_page, 20); + assert_eq!(offset, i64::MAX); + } + /// A 5-field crontab line is the most common way to get a schedule rejected, and both /// parsers report it in terms a crontab user cannot act on, so the seconds field and the /// equivalent expression must reach the caller for v1 and v2 alike. diff --git a/backend/windmill-common/src/webhook.rs b/backend/windmill-common/src/webhook.rs index 02d6f7ddc5..2f0d83f7e2 100644 --- a/backend/windmill-common/src/webhook.rs +++ b/backend/windmill-common/src/webhook.rs @@ -164,7 +164,10 @@ impl WebhookShared { reqwest::Client::builder() .connect_timeout(Duration::from_secs(5)) // TODO: investigate pool timeouts and such if TCP load is high - .timeout(Duration::from_secs(5)), + .timeout(Duration::from_secs(5)) + // The workspace webhook URL is SSRF-checked when saved; a redirect + // would send the POST past that check. + .redirect(reqwest::redirect::Policy::none()), ) .build() .unwrap(); diff --git a/backend/windmill-common/src/worker.rs b/backend/windmill-common/src/worker.rs index c742a3efcb..ad0a5068e8 100644 --- a/backend/windmill-common/src/worker.rs +++ b/backend/windmill-common/src/worker.rs @@ -41,6 +41,8 @@ use crate::{ #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Default)] pub struct CustomTags { pub global: Vec, + /// Keyed by the entry's name, which may hold placeholders like a global entry: only a name + /// without any can be looked up by the tag it admits, the others through [`custom_tag_matches`]. pub specific: HashMap, } @@ -83,20 +85,7 @@ impl CustomTags { } else { self.specific .iter() - .map(|(tag, tag_data)| { - let separator = tag_data.tag_type.corresponding_separator(); - let mut workspaces = tag_data - .workspaces - .iter() - .map(|w| w.to_string()) - .collect::>() - .join(&*separator.to_string()); - if tag_data.tag_type == SpecificTagType::AllExcluding { - // the AllExcluding tag syntax has a leading separator - workspaces.insert(0, separator); - } - format!("{}({})", tag, workspaces) - }) + .map(|(tag, tag_data)| tag_data.authored(tag)) .collect::>() }; let all_tags = self.global.clone(); @@ -104,6 +93,84 @@ impl CustomTags { } } +/// Whether a job whose tag resolved to `tag` falls under the custom tag `entry`. An entry holding +/// placeholders is a pattern over resolved tags: `$args[...]` and `$flow_expr[...]` match any +/// text, since whoever pushes the job picks their values, and `$workspace` matches +/// `tag_workspace`, what the job's own `$workspace` resolves to. The text around them must match +/// as written: it is what confines `gpu-$args[size]` to the `gpu-` tags. +/// +/// Placeholders whose values are tied, a repeat or one reading inside another, make an entry no +/// wildcard pattern describes (`t-$args[id]-$args[id]` never resolves to `t-a-b`). Such an entry +/// matches nothing here: it admits only a job whose tag is written exactly as the entry is. +pub fn custom_tag_matches(entry: &str, tag: &str, tag_workspace: &str) -> bool { + if !entry.contains('$') { + return entry == tag; + } + let dynamic: Vec<&str> = CUSTOM_TAG_PLACEHOLDER + .find_iter(entry) + .map(|m| m.as_str()) + .filter(|p| *p != "$workspace") + .collect(); + let tied = dynamic + .iter() + .enumerate() + .any(|(i, a)| dynamic[i + 1..].iter().any(|b| placeholders_tied(a, b))); + if tied { + return false; + } + // The literal runs between wildcards, with `$workspace` substituted. + let mut pieces = vec![]; + let mut current = String::new(); + let mut last_end = 0; + for m in CUSTOM_TAG_PLACEHOLDER.find_iter(entry) { + current.push_str(&entry[last_end..m.start()]); + if m.as_str() == "$workspace" { + current.push_str(tag_workspace); + } else { + pieces.push(std::mem::take(&mut current)); + } + last_end = m.end(); + } + current.push_str(&entry[last_end..]); + pieces.push(current); + + let [first, rest @ ..] = pieces.as_slice() else { + return false; + }; + let Some((last, middle)) = rest.split_last() else { + return tag == first; + }; + let Some(mut inner) = tag + .strip_prefix(first.as_str()) + .and_then(|t| t.strip_suffix(last.as_str())) + else { + return false; + }; + for piece in middle { + match inner.find(piece.as_str()) { + Some(i) => inner = &inner[i + piece.len()..], + None => return false, + } + } + true +} + +/// Whether two `$args[...]` / `$flow_expr[...]` placeholders read the same value, or one reads a +/// value inside the other's. +fn placeholders_tied(a: &str, b: &str) -> bool { + let (Some((kind_a, path_a)), Some((kind_b, path_b))) = (a.split_once('['), b.split_once('[')) + else { + return false; + }; + let (path_a, path_b) = (path_a.trim_end_matches(']'), path_b.trim_end_matches(']')); + let inside = |outer: &str, inner: &str| { + inner + .strip_prefix(outer) + .is_some_and(|rest| rest.starts_with('.')) + }; + kind_a == kind_b && (path_a == path_b || inside(path_a, path_b) || inside(path_b, path_a)) +} + /// Marker suffixed to a workspace id inside a custom tag's scope (`mytag(prod*)`) to extend the /// entry to that workspace's forks. `*` cannot appear in a workspace id (the `proper_id` check /// constraint restricts them to `^\w+(-\w+)*$`), so it can never collide with a real id. @@ -183,6 +250,22 @@ impl SpecificTagData { pub fn is_fork_scoped(&self) -> bool { self.workspaces.iter().any(|w| w.include_forks) } + + /// The entry named `name` with this scope, as written in the custom tags: `tag(ws1+ws2)`. + pub fn authored(&self, name: &str) -> String { + let separator = self.tag_type.corresponding_separator(); + let mut workspaces = self + .workspaces + .iter() + .map(|w| w.to_string()) + .collect::>() + .join(&*separator.to_string()); + if self.tag_type == SpecificTagType::AllExcluding { + // the AllExcluding tag syntax has a leading separator + workspaces.insert(0, separator); + } + format!("{}({})", name, workspaces) + } } #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] pub enum SpecificTagType { @@ -464,7 +547,9 @@ lazy_static::lazy_static! { - // ^([\w-]+) # Group 1: tag name + // ^( # Group 1: tag name, a pattern when it holds placeholders + // (?:[\w-]|\$workspace|\$(?:args|flow_expr)\[(?:\w+\.)*\w+\])+ + // ) // \( # Literal '(' // ( # Group 2: the full workspace list // (?:[\w-]+\*?\+)*[\w-]+\*? # NoneExcept pattern: ws1+ws2* @@ -473,8 +558,12 @@ lazy_static::lazy_static! { // ) // \)$ # Closing ')' // - // The optional `*` after each workspace id is the fork marker, see [`WorkspaceMatcher`]. - static ref CUSTOM_TAG_REGEX: Regex = Regex::new(r"^([\w-]+)\(((?:[\w-]+\*?\+)*[\w-]+\*?|(?:\^[\w-]+\*?)+)\)$").unwrap(); + // The placeholders are CUSTOM_TAG_PLACEHOLDER's. The optional `*` after each workspace id is + // the fork marker, see [`WorkspaceMatcher`]. + static ref CUSTOM_TAG_REGEX: Regex = Regex::new(r"^((?:[\w-]|\$workspace|\$(?:args|flow_expr)\[(?:\w+\.)*\w+\])+)\(((?:[\w-]+\*?\+)*[\w-]+\*?|(?:\^[\w-]+\*?)+)\)$").unwrap(); + + // The placeholders a job's tag is resolved from when it is pushed, see [`custom_tag_matches`]. + static ref CUSTOM_TAG_PLACEHOLDER: Regex = Regex::new(r"\$workspace|\$(?:args|flow_expr)\[(?:\w+\.)*\w+\]").unwrap(); pub static ref DISABLE_BUNDLING: bool = std::env::var("DISABLE_BUNDLING") .ok() @@ -3070,6 +3159,92 @@ mod tests { assert_eq!(result, vec!["foo", "legacy(^ws1^ws2)", "urgent(ws1+ws2)"]); } + #[test] + fn test_custom_tag_matches_resolved_tags() { + let matches = |entry, tag| custom_tag_matches(entry, tag, "ws1"); + + assert!(matches("gpu", "gpu")); + assert!(!matches("gpu", "gpu-large")); + + // The text around a placeholder fences what its value can make of the tag. + assert!(matches("gpu-$args[size]", "gpu-large")); + assert!(matches("gpu-$flow_expr[results.a.size]", "gpu-")); + assert!(!matches("gpu-$args[size]", "prod")); + assert!(!matches("gpu-$args[size]", "xgpu-large")); + assert!(matches("$args[region]-gpu", "eu-gpu")); + assert!(!matches("$args[region]-gpu", "eu-gpu-x")); + assert!(matches("a-$args[x]-b-$args[y]-c", "a-1-b-2-c")); + assert!(!matches("a-$args[x]-b-$args[y]-c", "a-1-c")); + assert!(!matches("ab$args[x]ba", "aba")); + + // A bare placeholder admits every tag. + assert!(matches("$flow_expr[results.a.tag]", "anything")); + + // Tied placeholders are no pattern: their entry admits only its own text. + assert!(!matches("t-$args[id]-$args[id]", "t-a-a")); + assert!(!matches("t-$args[a]-$args[a.b]", "t-x-y")); + assert!(matches("t-$args[a.x]-$args[a.y]", "t-x-y")); + assert!(matches("t-$args[id]-$flow_expr[flow_input.id]", "t-x-y")); + + // `$workspace` stands for the job's own workspace only. + assert!(matches("tag-$workspace", "tag-ws1")); + assert!(!matches("tag-$workspace", "tag-ws2")); + assert!(matches("$workspace-$args[size]", "ws1-large")); + assert!(!matches("$workspace-$args[size]", "ws2-large")); + } + + #[test] + fn test_scoped_custom_tag_patterns_parse_and_round_trip() { + let input = vec![ + "gpu-$args[size](ws1+ws2)".to_string(), + "cpu-$flow_expr[results.a.size](^ws1)".to_string(), + "$workspace-$args[x](prod*)".to_string(), + "t-$args[id]-$args[id](ws1)".to_string(), + ]; + let tags = CustomTags::from(input.clone()); + + assert!(tags.global.is_empty()); + assert_eq!( + tags.specific["gpu-$args[size]"], + SpecificTagData { + tag_type: SpecificTagType::NoneExcept, + workspaces: vec![matcher("ws1"), matcher("ws2")], + } + ); + assert_eq!( + tags.specific["cpu-$flow_expr[results.a.size]"], + SpecificTagData { + tag_type: SpecificTagType::AllExcluding, + workspaces: vec![matcher("ws1")], + } + ); + assert!(tags.specific["$workspace-$args[x]"].is_fork_scoped()); + assert!(tags.specific.contains_key("t-$args[id]-$args[id]")); + + let mut result = tags.to_string_vec(None); + result.sort(); + let mut expected = input; + expected.sort(); + assert_eq!(result, expected); + + let mut for_ws1 = tags.to_string_vec(Some(&chain(&["ws1"]))); + for_ws1.sort(); + assert_eq!(for_ws1, vec!["gpu-$args[size]", "t-$args[id]-$args[id]"]); + + // Only the placeholders a tag resolves from make a name a pattern: anything else is no + // scoped entry at all. + for literal in [ + "gpu-$foo(ws1)", + "gpu-$args[](ws1)", + "gpu-$args[a.](ws1)", + "gpu.x(ws1)", + ] { + let tags = CustomTags::from(vec![literal.to_string()]); + assert_eq!(tags.global, vec![literal]); + assert!(tags.specific.is_empty()); + } + } + #[test] fn test_fork_marker_parses_and_round_trips() { let tags = CustomTags::from(vec![ diff --git a/backend/windmill-common/src/workspaces.rs b/backend/windmill-common/src/workspaces.rs index 711b4dac29..6f82e3a261 100644 --- a/backend/windmill-common/src/workspaces.rs +++ b/backend/windmill-common/src/workspaces.rs @@ -184,7 +184,7 @@ pub enum ObjectType { DatatableMigration, } -pub const LATEST_GIT_SYNC_SCRIPT_PATH: &str = "hub/28969/sync-script-to-git-repo-windmill"; +pub const LATEST_GIT_SYNC_SCRIPT_PATH: &str = "hub/28971/sync-script-to-git-repo-windmill"; /// Hub script that applies a repository's state back into a workspace /// (the repo → Windmill / "pull" direction). Same script the UI runs from @@ -192,7 +192,7 @@ pub const LATEST_GIT_SYNC_SCRIPT_PATH: &str = "hub/28969/sync-script-to-git-repo /// ignores the slug, so the slug is kept free of characters that would be /// percent-encoded into the run URL (a `:` becomes `%3A`, which some hardened /// reverse proxies reject as double-encoding when the client re-encodes it). -pub const GIT_SYNC_PULL_SCRIPT_PATH: &str = "hub/28957/git-sync-init-repository-windmill"; +pub const GIT_SYNC_PULL_SCRIPT_PATH: &str = "hub/28970/git-sync-init-repository-windmill"; /// Prefix used to identify fork workspaces. A workspace whose id starts with this string is a /// fork of another workspace. @@ -1304,6 +1304,12 @@ pub struct DataTable { /// nothing local for a fork admin to widen. #[serde(default, skip_serializing_if = "Option::is_none")] pub reference: Option, + /// Set on a *clone* — a terminal entry whose database was copied from the entry this names. The + /// copy holds that entry's rows, so who may connect as which role stays that entry's decision: + /// the clone carries no `permissions` of its own and is governed like a pointer, while + /// connecting to its own database. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub governed_by: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub forked_from: Option, /// Whether the SQL-migrations feature is opted in for this data table. @@ -1362,9 +1368,16 @@ pub const DATATABLE_TENANT_WILDCARD: &str = "*"; /// enough to survive a fork of a fork. const DATATABLE_REFERENCE_MAX_DEPTH: usize = 20; -/// Exactly one of `database` and `reference` must be set. Called wherever an entry is persisted, -/// so nothing downstream has to handle an entry that is both or neither. +/// Exactly one of `database` and `reference` must be set, and a clone's `governed_by` leaves no +/// `permissions` beside it. Called wherever an entry is persisted, so nothing downstream has to +/// handle an entry that is both or neither, or a clone with a decision of its own. pub fn validate_datatable_shape(name: &str, dt: &DataTable) -> Result<()> { + if dt.governed_by.is_some() && (dt.database.is_none() || dt.permissions.is_some()) { + return Err(Error::BadRequest(format!( + "Data table '{name}' is a clone, which owns a database and takes its roles from the \ + data table it was cloned from" + ))); + } match (&dt.database, &dt.reference) { (Some(_), None) | (None, Some(_)) => Ok(()), (Some(_), Some(_)) => Err(Error::BadRequest(format!( @@ -1452,23 +1465,28 @@ pub async fn read_datatable_entry(db: &DB, w_id: &str, name: &str) -> Result(datatable.clone())?) } -/// The terminal entry a reference chain lands on: the workspace that governs the data table, the -/// entry name there, and the entry itself. A terminal entry resolves to itself. +/// The terminal entry a reference chain lands on, and what governs it: the workspace and name of +/// the entry that owns the database, the entry itself, and — for a clone — the entry its +/// `governed_by` chain lands on. A terminal entry that is not a clone resolves to itself and +/// governs itself. /// -/// Every decision downstream — which database to connect to, whose `permissions` apply, whose -/// members tenants are evaluated against, who may administer it — is taken on this, never on the -/// entry the caller named. +/// Every decision downstream is taken on this, never on the entry the caller named. Which database +/// to connect to comes from `workspace_id` / `name` / `datatable.database`; whose `permissions` +/// apply (already in `datatable.permissions`), whose members tenants are evaluated against and who +/// may administer it come from [`GoverningDatatable::governing_workspace_id`]. /// /// Authorization: resolving deliberately crosses into the governing workspace, so it answers for a /// workspace the caller may not belong to and checks nothing itself. It is the input to the /// checks, not one of them: callers MUST pass what it returns to /// [`can_use_datatable_role_in_governing_workspace`] or [`ensure_datatable_admin_access`] before -/// acting on it, and MUST NOT return its `permissions` or `workspace_id` to a caller from +/// acting on it, and MUST NOT return its `permissions` or workspace ids to a caller from /// elsewhere without gating on the answer. pub struct GoverningDatatable { pub workspace_id: String, pub name: String, pub datatable: DataTable, + /// For a clone, the entry whose `permissions` govern it. `None` when the entry governs itself. + pub governor: Option, } impl GoverningDatatable { @@ -1480,6 +1498,13 @@ impl GoverningDatatable { .as_ref() .is_some_and(|d| d.resource_type == DataTableCatalogResourceType::Instance) } + + /// The workspace whose admins administer the data table and whose members its tenants are. + pub fn governing_workspace_id(&self) -> &str { + self.governor + .as_ref() + .map_or(&self.workspace_id, |g| &g.workspace_id) + } } pub async fn resolve_governing_datatable( @@ -1490,6 +1515,9 @@ pub async fn resolve_governing_datatable( let mut workspace_id = w_id.to_string(); let mut name = name.to_string(); let mut hops = 0; + // The clone a `governed_by` chain started from: it keeps its database, and takes the + // `permissions` of wherever the chain lands. + let mut clone: Option<(String, String, DataTable)> = None; for _ in 0..DATATABLE_REFERENCE_MAX_DEPTH { let datatable = read_datatable_entry(db, &workspace_id, &name) .await @@ -1500,21 +1528,48 @@ pub async fn resolve_governing_datatable( // A pointer outlives the workspace it names: deleting one only nulls the fork // lineage, it does not sweep the entries that pointed at it. Say which one is // gone rather than reporting a data table this workspace never had. - Error::NotFound(format!( - "Data table '{name}' of workspace '{workspace_id}' governs this one and no \ - longer exists. A superadmin can point this data table somewhere else." - )) + if clone.is_some() { + Error::NotFound(format!( + "Data table '{name}' of workspace '{workspace_id}', which this clone \ + takes its roles from, no longer exists, so nobody is let into the copy." + )) + } else { + Error::NotFound(format!( + "Data table '{name}' of workspace '{workspace_id}' governs this one and \ + no longer exists. A superadmin can point this data table somewhere \ + else." + )) + } } })?; hops += 1; validate_datatable_shape(&name, &datatable)?; - match &datatable.reference { - None => return Ok(GoverningDatatable { workspace_id, name, datatable }), - Some(reference) => { - workspace_id = reference.workspace_id.clone(); - name = reference.datatable.clone(); + let next = match (&datatable.reference, &datatable.governed_by) { + (Some(reference), _) => reference.clone(), + (None, Some(governed_by)) => { + let governed_by = governed_by.clone(); + if clone.is_none() { + clone = Some((workspace_id.clone(), name.clone(), datatable)); + } + governed_by } - } + (None, None) => { + return Ok(match clone { + None => GoverningDatatable { workspace_id, name, datatable, governor: None }, + Some((clone_w_id, clone_name, mut clone_datatable)) => { + clone_datatable.permissions = datatable.permissions; + GoverningDatatable { + workspace_id: clone_w_id, + name: clone_name, + datatable: clone_datatable, + governor: Some(DataTableReference { workspace_id, datatable: name }), + } + } + }); + } + }; + workspace_id = next.workspace_id; + name = next.datatable; } Err(Error::BadRequest(format!( "Data table '{name}' points at another data table through more than \ @@ -1554,17 +1609,20 @@ pub async fn resolve_workspace_governing_datatables( let mut entries = Entries::new(); let listed = load(db, &[w_id.to_string()], &mut entries).await?; - // (index into `listed`, workspace, entry name) still to be followed. - let mut cursors: Vec<(usize, String, String)> = listed + // (index into `listed`, workspace, entry name, the clone the chain started from) still to be + // followed. A clone keeps its own database and takes the `permissions` of wherever its + // `governed_by` chain lands, as the single resolution does. + type Clone = Option<(String, String, DataTable)>; + let mut cursors: Vec<(usize, String, String, Clone)> = listed .iter() .enumerate() - .map(|(i, name)| (i, w_id.to_string(), name.clone())) + .map(|(i, name)| (i, w_id.to_string(), name.clone(), None)) .collect(); let mut resolved: Vec<(usize, GoverningDatatable)> = vec![]; for _ in 0..DATATABLE_REFERENCE_MAX_DEPTH { let mut next = vec![]; - for (i, ws, name) in cursors.drain(..) { + for (i, ws, name, clone) in cursors.drain(..) { let Some(value) = entries .get(&ws) .and_then(|m| m.get(&name)) @@ -1578,14 +1636,41 @@ pub async fn resolve_workspace_governing_datatables( if validate_datatable_shape(&name, &datatable).is_err() { continue; } - match &datatable.reference { - None => { - resolved.push((i, GoverningDatatable { workspace_id: ws, name, datatable })) - } - Some(reference) => next.push(( + match (&datatable.reference, &datatable.governed_by) { + (Some(reference), _) => next.push(( i, reference.workspace_id.clone(), reference.datatable.clone(), + clone, + )), + (None, Some(governed_by)) => { + let (governor_ws, governor_name) = + (governed_by.workspace_id.clone(), governed_by.datatable.clone()); + let clone = clone.or(Some((ws, name, datatable))); + next.push((i, governor_ws, governor_name, clone)); + } + (None, None) => resolved.push(( + i, + match clone { + None => GoverningDatatable { + workspace_id: ws, + name, + datatable, + governor: None, + }, + Some((clone_ws, clone_name, mut clone_datatable)) => { + clone_datatable.permissions = datatable.permissions; + GoverningDatatable { + workspace_id: clone_ws, + name: clone_name, + datatable: clone_datatable, + governor: Some(DataTableReference { + workspace_id: ws, + datatable: name, + }), + } + } + }, )), } } @@ -1594,7 +1679,7 @@ pub async fn resolve_workspace_governing_datatables( } let to_load: Vec = next .iter() - .map(|(_, ws, _)| ws.clone()) + .map(|(_, ws, _, _)| ws.clone()) .filter(|ws| !entries.contains_key(ws)) .collect::>() .into_iter() @@ -1967,6 +2052,8 @@ pub fn strip_datatable_permissions( /// As [`parse_datatable_ref`], except that an entry whose stored name itself contains `?` — which /// names could before they were restricted — resolves by that exact name, without a role. It is /// looked up first, so `sales?role=x` never reaches a different entry than the one stored so. +/// When `sales` is stored too, the reference means either one, and is refused rather than +/// resolved to whichever is looked up first. /// /// Authorization: checks nothing, and its answer reveals whether `w_id` stores that exact name. /// Callers MUST already act for `w_id` — a job of it, or a caller authenticated into it — and @@ -1977,16 +2064,26 @@ pub async fn parse_datatable_ref_for( reference: &str, ) -> Result<(String, Option)> { if reference.contains('?') { - let exists = sqlx::query_scalar::<_, Option>( - "SELECT (datatable->'datatables') ? $2 FROM workspace_settings WHERE workspace_id = $1", + let role_target = parse_datatable_ref(reference) + .ok() + .and_then(|(name, role)| role.map(|_| name)); + let (exists, target_exists) = sqlx::query_as::<_, (Option, Option)>( + "SELECT (datatable->'datatables') ? $2, (datatable->'datatables') ? $3 + FROM workspace_settings WHERE workspace_id = $1", ) .bind(w_id) .bind(reference) + .bind(role_target) .fetch_optional(db) .await? - .flatten() - .unwrap_or(false); - if exists { + .unwrap_or((None, None)); + if exists.unwrap_or(false) { + if let (Some(name), Some(true)) = (role_target, target_exists) { + return Err(Error::BadRequest(format!( + "Data table reference '{reference}' names both the data table '{reference}' \ + and a role on the data table '{name}'. Rename '{reference}' to use either." + ))); + } return Ok((reference.to_string(), None)); } } @@ -3344,6 +3441,7 @@ mod tests { resource_path: "dt_main".to_string(), }), reference: None, + governed_by: None, forked_from: None, migrations_enabled: None, permissions: None, diff --git a/backend/windmill-common/tests/fixtures/guest_jwt_rsa8192_private.pem b/backend/windmill-common/tests/fixtures/guest_jwt_rsa8192_private.pem new file mode 100644 index 0000000000..bb443cf6bf --- /dev/null +++ b/backend/windmill-common/tests/fixtures/guest_jwt_rsa8192_private.pem @@ -0,0 +1,100 @@ +-----BEGIN PRIVATE KEY----- +MIISQgIBADANBgkqhkiG9w0BAQEFAASCEiwwghIoAgEAAoIEAQC6deMC6KWwkWV2 +9xb4bXLt6ceNGQD3gJH3sYqpulWsZB0L3TdqXwnzL3C/8Uf3K8hLZ4Tw+llHCxaS +Z7aTEYyeLDJuYuJPVS8DAgVnXDluoDg7B1UtvoUa1/MjLQm4f/Qp322i7l0SH9kx +NRb7UZp/xx1tL/kiq9NIv1inK4ZC0DbleybBOoLlwP9Zqhazw3MJdZAyey4FAXTx +bZcjG3UtiZ2COXVP1qdtqhLrQQQsicoLqZuxAfXdIU6xgcYgGapVQNTVbo7w8pHn ++6RzlPb9rqyBs2IiKLFSb467mtZeM9dfbyD4DICy42O/JKAbTZw/f3trbs+tu+sK +rj7ysk+lGzuXj2/irOuvOSbzpxERdZuDyLOhlMej5+aJXkrmVhVPf/3wt/6oKCzo +7Ta24OvGaz6pxK1iDOQvzkCqJpaTjSfpcscHyU3ztXCUINtsq7s78XoOxJZkerfq +LDxuutYviFiB9pQDzYdErL8XMUkRXXY22hpnLHt8GV+lNCJs2KxcECzkgwfjIn9X +Y66U4iO+k9EvY/FHyI25ur0eKWDnJAPL6PypVhx9NOuBkot6nwUugzWzM/MKJ9Hf +1n9pwVNlMam8zgMQ5Ivu3+RiOV3P0iuAaw7d1sZRSSX11l/EyjGFDpwmnFNjleEK +vnop6rCftWAsd2XJHXiD1/80QIgKJHdflpHOcy6Su5w/CRYc/jC2rJjUr9W+hBwg +fnayip8MTMa4cAXzbFmSc5NCzSsqHkVq/4RdfdMephubGmWcPDQnkUf/n/RvoZNi +VktW34lX3YIe97600vSWPfWahj/EUnfI8m+4Qzuz2o6N9basCNftKPDIc5ej3PMS +RVIgBhEXUh1wVFdEUs8RKHg/QV0xrN2qbKcS8XvV3fFfLuFZo4a66nZ5zgGGJUgL +ED41cEqf/sQObfhGgIw25v/315ARiv9thRF3tUMLGQZNQ8tSs2VMuU6szBQciplx +y2y4JzXdv+G0PvheE7tVdsGsMipSHTZDdNZo51MOBSgsNTyNmWMZlsfoZTTZ2o2y +XtKv8xUP01KnlwR6Z8hY1CAPxM1QKDU5nKp9T/df6GEhohps9gSt6cs6SHlH8piq +pyR0xiDvS+vcPN/IbOaRgE9WmfM5py72SNZbetiUHdUTmAC1dd0IH3opte1uVpQQ +sauKVWZuzV8Bcd2O06QFOVJDwoxm00YaF0rqhaisppah5eQMN/ctlqWVDHXR8laV +CNkzWYE/jxAU9dGuCMCIZ7BSs5U9mZ7pWstagXWEFTI9IugjvNnaZDijZQJSPmAZ +gQtAxH/z8R9e/PihW9dDQc/nKBC9Lpta3oFGq74fc7Y+Wu/Uskjf8qtFOOVrvfZx +kwlRR0s5AgMBAAECggQAEEeerHBXlxT6wOpN7R7RPAh/TchzFyj4laHAs5T7SfPr +vU9xOS5XFg58T6XnXKNL95lwOAIDrXpAXEQF4s1Tc7P0K0Mi8z5CvoZ8pkudEBWZ +Ez5Op5HM2U0lg1q5UIyLauSBFAqvwZdaRTuXaBX14VYnMZNQs0BBUEa0nNPmAwte +36Zi+df7+cdiBBgfp05XSuoizuWGXRWFelbRLVWGY5VyVR5jKbdaIu8Ie9XPYXwe +UepbF5dOOiJgVos4GzFid9MTQhdHD65+Ir0L3LOS5Q9NLH3XAZh/8uyej66zLMJl ++YUgYcflrLjByryN5hReogSlZ/ni1A5mNSaFhHchEQ9Roqf5YtcqNMBOsFNT4Cxe +iaEKXf85dwwWSsQa0gWOEsTgDRZje5cWZLfhYRI+Q7K1Y+kyIi9x9OKTLsfzuapH +Y9CLpBhlX/Plgmlj/xb3YAJzX7q2TsuQJzf5hsl1Te0fnSlEPxjAyLQ0cvfwoz1/ +xsZAK+XsWid0/F2S6+H3GoqnAMSX+bLXX2/qnEEqUcdiKjsVo+jUxvSQ8oM2XLf+ +RN1wwfAsyJnIPiXnW6tqrmbyV1Uf0TEs19hIg5lHiLxPT5bFGcJ6nT6bGCBeGnkF +J08xmxjXc91pHQ+8iNgGftGJvK1FcppxbZi33dUJ2sfhP8Ck+GxES26eb5d6yTfD +oZRQmESvikHPU5shm+YbjPrYhEbEgLZKO3wUhCAvoFLN2TG0i2v//aaCSI/XfNWt +1XyE+uN/o6kQEPvaUy/0uHBuzoC4pZ8or22ywmeftCOHuT9YE9LIJuU81qMJvjP7 +BJ6YkcTO6G89iD0g6HHYRiD9I+pgih8B1NPq4VmFuJrvaqgAX9srmmMt7FXBrcqa +JSyLS+WeHTLlOiHN1DqOgpgq0gq6vC1i53xPqSUkFzAKpDOWgGNMVAABFN9EOVmM +zvPnJThz5PHDAuud9eaLjZ0V76IVXmRTz22warkdKhBW5BzDzy+zNp8mF5SMH9mZ ++JQvCGTc2itiKGKMDzvY99pBzHRlrlFL7n5SHLqxclBfnEJcU9wPvFyZ+6k8MGim +0kFrK63NqiceNWp5vdU/6V+boCYoFG8PDbABUxCbb+lhtnYx5AUcKfKX74Uv4gpX +AM0gsaY8U+4ATJbgoCTqplTKvXtyw4zcpJAXCMnhygKyDKNOk+Lckfquwx9r/TQN +hbbjjvwgBkzJtAlylCVYYEJ/03lazRpGR5osCksECxNU4TMDRD2Hd+zwUxdeRunm +7EusRSrKq8swi8LFyV7E8ISwwXegqVarHqXjCHhcn/5Ozw6PkVk39Ug2UE4ibesC +MzVP3lfP67gXgGCzYrImIUuqHGTyLFs8/B8+2OmdyQKCAgEA6PCozdeeQS0P/+ah +E015KECxB0aGbrvdpczXX5szHYb8KhejvHmVV2OLFisL7GyMeBhEZXAeKtMZwd66 +FMORgdFACpckR/a/aVEnebHx5fOKtAQfhLLH/3YE5dFIWxu9SKNZBrX0lIBFXfoK +lZYM7cGm7vN55AK99bYfCAqFs+HI9sDQLjbaxR41UOfZwPjA9VjBQ1vBr0N3UoOT +xf8Rw6hlAd+ED7iJavn6KUb64D5eB4eGSZNXM5HCp2ZtUFN/4bU5lbHPCSK29+Wd +mM5flZtIKQ7PzGyXqv4Kbtz9o1hPOQPeH0faDEcYJLHV/LAxHc1AkqJDXU8HANhI +m/XI44RlJesNntOdJuCumHtTAsreWAYoydXurOWHkDIpx6OLi0F3eJrOV6dhwCCG +v3bX4PIBjYZKyfHukcoCzGetXpu5bDKC1uwwZLJmF/QTO3AeVpWNrbWkBfj/PCio +8RodjVlPkDaoCrDmpca22fNo+93IM2m1SWdSnTlXvgg58cLUrlMUbg24Lo1cyySF +1AnRw0uvaFLdH8iMtKZIA3zgCZyUKV1DcGUKsdcL/9IYD6EO5bCcA6p5gcxxBOan +6MEHIuJKycmZMGutN5rm3uDHQgmNCs25ma3fWQF0s/dRJ2lg947qKUKevqBkd4gt +BuB8VXUGsHXGDWHfz/eLjWYA3k8CggIBAMzrTqnmT27U1SXPwEXI0kc6A44D8skD +a4rqxOMBTGvu+HeVdEBMcxYnGeJV2QtZY96KXwadiWuqdEr4Pc7JWnAa2f9yWNCb +QI0YJSWSNcDuPsSV1K15MM7uxKk/oNcXdbjrAfVLzCmEYDg085QV80rl7Q6WRaYv +acvipeZY82LuGhtAJAN6S8VWmDqZ5vBDd61F09AlBrpIFoSBRnYVkEwaXXwFJauz +S+kEmCIb97tBf7nr08ldoNvDdecRtm+s9odfUhm+ONroZf7OEyTCOXB666oDogpI +hktuliNUVpXeZ5QtkzpxP2SBbHNWZ9LiAGpTv2wa7z7k1PqCVR6ABGMkTkge41iy ++v82KYi4qRnUfVlFcn9xLyBDaaCWJYrsux8OmAW48cVN1Og0tKu/T+bFUvHs0etZ +oijYoYZaf9ERO0X0tuMXKIuT0QcPnkcT+KXFY6uzOC1UmF5ITCeKogGwzkFbvInA +w8OiFYBkgXZ41tE/oOARm1AhuaNc/ZfWujLECjuJgvw0uhdUyihGw8ZJv91WiIjX +Jqq1BNrhvDQqTPSnDvhcBBDGdQA6HJRGAerIx7PgCKzS2WbrfmsFzjfZ3zbrUSNf +e8CM37aDzte25g7HmvFuwOLmE9DXNnE+cWGpb/S1l4ACahnqKF9zj3uKVpN1kG+l +xFH+ADFgNCP3AoICAQDW+fuf9QVdezwP3IX9bXuKVaHLaB/7JDBbqt99oZsb/g7M +vgJjmAXmn7ymGdYoPMmqsCK3RzDgLvDPhkFPlmm8WCR1YaWfMK7YgIgg/MOViB0T +I9b+vYxsTDwa6PmQ89nNpssrsVNmlbbvEysfgFqwNg9c6aEteAZeNNUQs71PFOpG +KA45CHEAiG+GGmNRf4F9LIs4If9vQ6Fkqn//s/ZuDwQhJWoYpCNWchuYaqRvvb9D +zaR2VJ+evT/bBGGQR293G6xoZa//62TXBbdbZeK7DJMdKUlHjVSv/0IbrKH8VSzQ +HqypS8OFnzs96k3lpam+qkWUzmHDRILfGbxxCvV6XB6JTxPD5+JaFfOsT+x9YEKx +FAYit+hM8owY3OLIFzxaqXGcAaq7JIPHL32MSB7VwCMEvNJa64dUTw4kHO7YqHkI +zNkJhvLmi+pC4wO40DF/kQgvL7qPtIvBtnisAoERiUIvgvG05CwJBIqOPOE/yR/R +qxYfB1csONasT6VCZJ+pvozVQJ4hgF8KL8u+s80+IqRgcFEY2xpYQjml1mJhHHt0 +bVkm6mljVWcCkp+YFPjNrB9gFr2Kt+X6MvD1kueGHzgMYJybHlS/Q5scSHA8ltnv +BYvrs5az6u8MhAwGhm9yKRgcOhqNLTNrSWbcuOKmb5lb02WTxicrJHdLaL8WawKC +AgAvXHdK51MWj5RP7+tFdbu8AJmPujPj56l0avxkZwltrMa6hM1giwrSZOkjmanX +aKqfZaW+1qxLORM1vJg0ce84F8yosJn8ZNWPhrQXMerAk4/oSaTlDyyLtz22zGD1 +NZPc0FwcgI9ITTMb6okdgdFAT1Hx3CNu3Gh8Wf7z9T0TcWqRP8rTDvHyQyhd6vIo +N7+ySYBCDDy40y55I5PQeP1nyXQv1NS3EXKLAafTwlkIa4QEyY+ia8Rg5pUaecqL +lxNHiG1mS8PCPJJOSPy9szIpYA+VtM8/xzFav81tjvGQI94Wrk3n5QK8zPju01qd +r+RersqARt1i0Oq0l2jq6rWY4GxDWPg8qOlAERmaYuuab7Uk69shv1wf/dSs6+bz +a/F3pRzXvb4LVwjNPlk/lLD36kVYat69oXLMM/1pGwONo4jqizu10sVpNRnLov12 ++hOu5Kc7qOlj+LhamiUXwn/dGishif1SbT4RuiV0DkapxhYIOMeK4ibeZ10Ld9OY +w2U3lH208vxWJC2adNtgYbKf4q24j2ZxEiD6tWnQ2PlxqREuPmpEVqgN5IBoNAYk +8uVAiXU2iqIWp4jds7H9NFNML0va2ppfLZNhlm2iNmaZDsmvQsz53howlvKfgw4v +MDpuUwXwLgQ87tHs+WSpYwEXkYqe/D4Mv+hW9vi+cLi8UwKCAgAsehP6DOrt9bA3 +pVdtHJG1a9fm/rXC9TQS2oSE2paXRZpxhbk917Gdl2njo7tnoGXXBpQOb51/HUXX +QTARdp2LP1VFn9RJP3CwaK+Sxs1IBa9r1/p2ePDR85QR1HxN+Gj98EcEfKl1RSe6 +RmLOkLIg4pkNrDhwWyYJ1H5//d/yUQlZ015fQAKaHCdaoz7L3+0jOyFXDGrDr+Gc +680SyVqPfRx5dAm8TrfyXW6Q2uZEhIqxbcxTdG98Pxi0qLLAS/nD16IHC1iljHsI +QrfUvFvGg5v7kxeXrP0VPMRM0/Y8vno3Bbi04I0Pk3Wko8UhWuiUMjs+TVbEPKox +8zG/G+R5EpHKDxNyfUUwZ9+lyjFmBZOcVd3dze23VQYCzV/W04yiQdEl2C/sED2m +NycJ3GN/ByqpJJZY3u+B/WAiHPLe1ivIEeIjwdBt2w/3L/rKoRBR/P8LSRXLIIh4 +Xk1BMNygJxHTwsvFGhflx89+1BO/n8HrR0s/2G1QanxGdgso9bwmXMDrmJ+XxrUY +qBmDCU8RkBdUHKOuJlWWn2JaIswsEE5kAk7MgNNaBTv91VBYCDs50NW8VBaKnJjN +j+hXI80qKi7WqeoonFKFLDgUYbKnmzKsX+T0x/H99v+7Or5PmAPN+hRwj54/LEVR +MsjrTlnoCGXKgPQQ5ItWviGjqoCTiQ== +-----END PRIVATE KEY----- diff --git a/backend/windmill-common/tests/fixtures/guest_jwt_rsa8192_public.pem b/backend/windmill-common/tests/fixtures/guest_jwt_rsa8192_public.pem new file mode 100644 index 0000000000..44645f5af2 --- /dev/null +++ b/backend/windmill-common/tests/fixtures/guest_jwt_rsa8192_public.pem @@ -0,0 +1,25 @@ +-----BEGIN PUBLIC KEY----- +MIIEIjANBgkqhkiG9w0BAQEFAAOCBA8AMIIECgKCBAEAunXjAuilsJFldvcW+G1y +7enHjRkA94CR97GKqbpVrGQdC903al8J8y9wv/FH9yvIS2eE8PpZRwsWkme2kxGM +niwybmLiT1UvAwIFZ1w5bqA4OwdVLb6FGtfzIy0JuH/0Kd9tou5dEh/ZMTUW+1Ga +f8cdbS/5IqvTSL9YpyuGQtA25XsmwTqC5cD/WaoWs8NzCXWQMnsuBQF08W2XIxt1 +LYmdgjl1T9anbaoS60EELInKC6mbsQH13SFOsYHGIBmqVUDU1W6O8PKR5/ukc5T2 +/a6sgbNiIiixUm+Ou5rWXjPXX28g+AyAsuNjvySgG02cP397a27PrbvrCq4+8rJP +pRs7l49v4qzrrzkm86cREXWbg8izoZTHo+fmiV5K5lYVT3/98Lf+qCgs6O02tuDr +xms+qcStYgzkL85AqiaWk40n6XLHB8lN87VwlCDbbKu7O/F6DsSWZHq36iw8brrW +L4hYgfaUA82HRKy/FzFJEV12NtoaZyx7fBlfpTQibNisXBAs5IMH4yJ/V2OulOIj +vpPRL2PxR8iNubq9Hilg5yQDy+j8qVYcfTTrgZKLep8FLoM1szPzCifR39Z/acFT +ZTGpvM4DEOSL7t/kYjldz9IrgGsO3dbGUUkl9dZfxMoxhQ6cJpxTY5XhCr56Keqw +n7VgLHdlyR14g9f/NECICiR3X5aRznMukrucPwkWHP4wtqyY1K/VvoQcIH52soqf +DEzGuHAF82xZknOTQs0rKh5Fav+EXX3THqYbmxplnDw0J5FH/5/0b6GTYlZLVt+J +V92CHve+tNL0lj31moY/xFJ3yPJvuEM7s9qOjfW2rAjX7SjwyHOXo9zzEkVSIAYR +F1IdcFRXRFLPESh4P0FdMazdqmynEvF71d3xXy7hWaOGuup2ec4BhiVICxA+NXBK +n/7EDm34RoCMNub/99eQEYr/bYURd7VDCxkGTUPLUrNlTLlOrMwUHIqZcctsuCc1 +3b/htD74XhO7VXbBrDIqUh02Q3TWaOdTDgUoLDU8jZljGZbH6GU02dqNsl7Sr/MV +D9NSp5cEemfIWNQgD8TNUCg1OZyqfU/3X+hhIaIabPYErenLOkh5R/KYqqckdMYg +70vr3DzfyGzmkYBPVpnzOacu9kjWW3rYlB3VE5gAtXXdCB96KbXtblaUELGrilVm +bs1fAXHdjtOkBTlSQ8KMZtNGGhdK6oWorKaWoeXkDDf3LZallQx10fJWlQjZM1mB +P48QFPXRrgjAiGewUrOVPZme6VrLWoF1hBUyPSLoI7zZ2mQ4o2UCUj5gGYELQMR/ +8/EfXvz4oVvXQ0HP5ygQvS6bWt6BRqu+H3O2Plrv1LJI3/KrRTjla732cZMJUUdL +OQIDAQAB +-----END PUBLIC KEY----- diff --git a/backend/windmill-native-triggers/src/handler.rs b/backend/windmill-native-triggers/src/handler.rs index 27a50ba07a..2c785a4e26 100644 --- a/backend/windmill-native-triggers/src/handler.rs +++ b/backend/windmill-native-triggers/src/handler.rs @@ -16,7 +16,8 @@ use serde::{Deserialize, Serialize}; use sqlx::PgConnection; use std::sync::Arc; use windmill_api_auth::{ - check_scopes, create_token_internal, require_is_writer, ApiAuthed, NewToken, + build_scope_path_filter, check_scopes, create_token_internal, require_is_writer, ApiAuthed, + NewToken, }; use windmill_audit::{audit_oss::audit_log, ActionKind}; use windmill_common::{ @@ -701,6 +702,7 @@ async fn list_native_triggers_handler( query.per_page, query.path.as_deref(), query.is_flow, + build_scope_path_filter(&authed, "native_triggers", "read"), ) .await?; tx.commit().await?; diff --git a/backend/windmill-native-triggers/src/lib.rs b/backend/windmill-native-triggers/src/lib.rs index 570319ee0e..6398e8a6d9 100644 --- a/backend/windmill-native-triggers/src/lib.rs +++ b/backend/windmill-native-triggers/src/lib.rs @@ -56,7 +56,7 @@ use windmill_queue::PushArgsOwned; #[cfg(feature = "native_trigger")] use windmill_oauth::{ErrorField, ExecuteError, OClient, RefreshToken, Url, OAUTH_HTTP_CLIENT}; -use windmill_api_auth::ApiAuthed; +use windmill_api_auth::{ApiAuthed, ScopePathFilter}; pub mod handler; pub(crate) mod lock; pub mod sync; @@ -1493,9 +1493,17 @@ pub async fn list_native_triggers<'c, E: sqlx::Executor<'c, Database = Postgres> per_page: Option, path: Option<&str>, is_flow: Option, + scope: ScopePathFilter, ) -> Result> { let offset = (page.unwrap_or(0) * per_page.unwrap_or(100)) as i64; let limit = per_page.unwrap_or(100) as i64; + // In the WHERE, not a retain after the fetch: the result is paginated, and a + // post-fetch filter would let a page's size report how many rows the token + // may not read. + let (scope_all, scope_exact, scope_prefix) = match scope { + ScopePathFilter::AllowAll => (true, Vec::new(), Vec::new()), + ScopePathFilter::Restricted { exact, prefix } => (false, exact, prefix), + }; let triggers = sqlx::query_as!( NativeTrigger, @@ -1532,7 +1540,12 @@ pub async fn list_native_triggers<'c, E: sqlx::Executor<'c, Database = Postgres> WHERE f.workspace_id = nt.workspace_id AND f.path = nt.script_path )) - ) + ) AND + ( $7 + OR nt.script_path = ANY($8) + OR EXISTS ( SELECT 1 FROM unnest($9::text[]) AS pfx + WHERE nt.script_path = pfx + OR left(nt.script_path, length(pfx) + 1) = pfx || '/' ) ) LIMIT $3 OFFSET $4 "#, @@ -1541,7 +1554,10 @@ pub async fn list_native_triggers<'c, E: sqlx::Executor<'c, Database = Postgres> limit, offset, path, - is_flow + is_flow, + scope_all, + &scope_exact[..], + &scope_prefix[..], ) .fetch_all(db) .await?; diff --git a/backend/windmill-native-triggers/src/sync.rs b/backend/windmill-native-triggers/src/sync.rs index bc259f048e..4f75845429 100644 --- a/backend/windmill-native-triggers/src/sync.rs +++ b/backend/windmill-native-triggers/src/sync.rs @@ -214,8 +214,17 @@ pub async fn sync_workspace_triggers( workspace_id ); - let windmill_triggers = - list_native_triggers(db, workspace_id, T::SERVICE_NAME, None, None, None, None).await?; + let windmill_triggers = list_native_triggers( + db, + workspace_id, + T::SERVICE_NAME, + None, + None, + None, + None, + windmill_api_auth::ScopePathFilter::AllowAll, + ) + .await?; if windmill_triggers.is_empty() { tracing::info!( diff --git a/backend/windmill-queue/src/jobs.rs b/backend/windmill-queue/src/jobs.rs index 429701f7e0..50eb9c8b1f 100644 --- a/backend/windmill-queue/src/jobs.rs +++ b/backend/windmill-queue/src/jobs.rs @@ -1285,7 +1285,25 @@ async fn commit_completed_job( } let mut _skip_downstream_error_handlers = false; - tx = delete_job(tx, &job_id).warn_after_seconds(10).await?; + let (ntx, canceled_at_delete) = delete_job(tx, &job_id).warn_after_seconds(10).await?; + tx = ntx; + // `canceled_by` is only what the worker last read from the queue row. Deleting the row waits + // for a cancel still being written, so the row it removed is the final word on whether this + // job was canceled. + if canceled_by.is_none() { + if let Some(canceled) = canceled_at_delete { + sqlx::query!( + "UPDATE v2_job_completed SET status = 'canceled'::job_status, canceled_by = $2, \ + canceled_reason = $3 WHERE id = $1", + job_id, + canceled.username, + canceled.reason, + ) + .execute(&mut *tx) + .warn_after_seconds(10) + .await?; + } + } // tracing::error!("3 {:?}", start.elapsed()); if completed_job.is_flow_step() { @@ -1611,6 +1629,22 @@ async fn restart_job_if_perpetual_inner( }; if restart { + // Not `canceled_by`: a worker reads the queue row on a widening interval, up to every 5s + // once a job has run for a minute, so a cancel landing after its last read reaches a + // completion carrying none. `commit_completed_job` records it from the queue row it + // deletes, so the completed row is what says whether this loop was stopped. + let canceled = sqlx::query_scalar!( + "SELECT canceled_by IS NOT NULL AS \"canceled!\" FROM v2_job_completed \ + WHERE id = $1 AND workspace_id = $2", + queued_job.id, + &queued_job.workspace_id + ) + .fetch_optional(db) + .await? + .unwrap_or(false); + if canceled { + return Ok(()); + } let tx = PushIsolationLevel::IsolatedRoot(db.clone()); // perpetual jobs can run one job per 10s max. If the job was faster than 10s, schedule the next one with the appropriate delay @@ -4696,6 +4730,72 @@ pub fn interpolate_args(x: String, args: &PushArgs, workspace_id: &str) -> Strin } } +/// The queue an explicit `tag` sends a job pushed with `args` to, or `None` when `push` drops the +/// tag and the job runs on its default one. +pub async fn resolve_push_tag( + tag: &str, + args: &PushArgs<'_>, + workspace_id: &str, + db: &DB, +) -> Option { + // The flow runtime resolves a step's `$flow_expr[...]` before pushing it, so one still here + // was pushed with no flow state to read (a step test, a dependency job) and would name a + // queue no worker serves: the job runs on its default tag instead. + if tag.is_empty() || tag_reads_flow_expr(tag) { + return None; + } + // `$workspace` must resolve the same way the default tags do, or an explicit tag and a default + // tag from the same workspace address two different worker pools. Resolving costs a lookup, + // so pay it only for tags that actually interpolate `$workspace`. + let tag_ws = if tag.contains("$workspace") { + crate::tags::tag_workspace_id(workspace_id, db).await + } else { + workspace_id.to_string() + }; + Some(interpolate_args(tag.to_string(), args, &tag_ws)) +} + +/// Refuses a `tag` the caller chose that the instance's custom tags do not let `w_id` use, +/// judging the queue it resolves to. `args` must be the ones the job is pushed with: resolving +/// with any others checks a queue the job does not land on. +pub async fn check_tag_available_for_push( + db: &DB, + w_id: &str, + tag: &str, + args: &PushArgs<'_>, + is_super_admin: bool, + scope_tags: Option>, +) -> Result<(), Error> { + check_tag_written_as_available_for_push(db, w_id, tag, tag, args, is_super_admin, scope_tags) + .await +} + +/// [`check_tag_available_for_push`] for a `tag` the flow runtime already partly resolved from +/// `written_tag`, the step's tag as its author wrote it. +pub async fn check_tag_written_as_available_for_push( + db: &DB, + w_id: &str, + written_tag: &str, + tag: &str, + args: &PushArgs<'_>, + is_super_admin: bool, + scope_tags: Option>, +) -> Result<(), Error> { + let Some(resolved_tag) = resolve_push_tag(tag, args, w_id, db).await else { + return Ok(()); + }; + windmill_common::jobs::check_tag_available_for_workspace_internal( + db, + w_id, + written_tag, + Some(&resolved_tag), + crate::tags::tag_workspace_id(w_id, db), + is_super_admin, + scope_tags, + ) + .await +} + pub fn fullpath_with_workspace( workspace_id: &str, script_path: Option<&String>, @@ -5087,34 +5187,44 @@ async fn extract_result_from_job_result( } } +/// Also reports the cancellation the deleted row carried, if any. Unlike a plain read of the queue +/// row, this waits for a cancel that is still being written, so it is the last word on one. pub async fn delete_job<'c>( mut tx: Transaction<'c, Postgres>, job_id: &Uuid, -) -> windmill_common::error::Result> { +) -> windmill_common::error::Result<(Transaction<'c, Postgres>, Option)> { #[cfg(feature = "prometheus")] if METRICS_ENABLED.load(std::sync::atomic::Ordering::Relaxed) { QUEUE_DELETE_COUNT.inc(); } otel_incr_queue_delete_count(); - let job_removed = - sqlx::query_scalar!("DELETE FROM v2_job_queue WHERE id = $1 RETURNING 1", job_id,) - .fetch_optional(&mut *tx) - .await; + let job_removed = sqlx::query!( + "DELETE FROM v2_job_queue WHERE id = $1 RETURNING canceled_by, canceled_reason", + job_id, + ) + .fetch_optional(&mut *tx) + .await; - if let Err(job_removed) = job_removed { - tracing::error!( - "Job {job_id} could not be deleted: {job_removed}. This is not necessarily an error, as the job might have been deleted by another process such as in the case of cancelling" - ); - } else { - let job_removed = job_removed.unwrap().flatten().unwrap_or(0); - if job_removed != 1 { - tracing::error!("Job {job_id} could not be deleted, returned not 1: {job_removed}. This is not necessarily an error, as the job might have been deleted by another process such as in the case of cancelling"); + let canceled = match &job_removed { + Err(job_removed) => { + tracing::error!( + "Job {job_id} could not be deleted: {job_removed}. This is not necessarily an error, as the job might have been deleted by another process such as in the case of cancelling" + ); + None } - } + Ok(None) => { + tracing::error!("Job {job_id} could not be deleted, no row was removed. This is not necessarily an error, as the job might have been deleted by another process such as in the case of cancelling"); + None + } + Ok(Some(row)) => row.canceled_by.as_ref().map(|username| CanceledBy { + username: Some(username.clone()), + reason: row.canceled_reason.clone(), + }), + }; tracing::debug!("Job {job_id} deleted"); - Ok(tx) + Ok((tx, canceled)) } pub async fn job_is_complete(db: &DB, id: Uuid, w_id: &str) -> error::Result { @@ -6535,23 +6645,9 @@ async fn push_inner<'c, 'd>( ); windmill_common::worker::dedicated_worker_tag(workspace_id, &full_path) } else { - // The flow runtime resolves a step's `$flow_expr[...]` before pushing it, so one still here - // was pushed with no flow state to read (a step test, a dependency job) and would name a - // queue no worker serves: the job runs on its default tag instead. - if tag == Some("".to_string()) || tag.as_deref().is_some_and(tag_reads_flow_expr) { - tag = None; - } - - // `$workspace` must resolve the same way the default tags below do, or an explicit tag and - // a default tag from the same workspace address two different worker pools. Resolving costs - // a lookup, so pay it only for tags that actually interpolate `$workspace`. let interpolated_tag = match tag { + Some(x) => resolve_push_tag(&x, &args, workspace_id, db).await, None => None, - Some(x) if x.contains("$workspace") => { - let tag_ws = crate::tags::tag_workspace_id(&workspace_id, db).await; - Some(interpolate_args(x, &args, &tag_ws)) - } - Some(x) => Some(interpolate_args(x, &args, workspace_id)), }; let effective_ws = per_workspace_tag(&workspace_id, db).await; diff --git a/backend/windmill-queue/src/schedule.rs b/backend/windmill-queue/src/schedule.rs index 5d5cf005fc..7a9a36ff19 100644 --- a/backend/windmill-queue/src/schedule.rs +++ b/backend/windmill-queue/src/schedule.rs @@ -19,7 +19,6 @@ use windmill_common::ee_oss::LICENSE_KEY_VALID; use windmill_common::flows::Retry; use windmill_common::get_flow_version_info_from_version; use windmill_common::get_latest_flow_version_id_for_path; -use windmill_common::jobs::check_tag_available_for_workspace_internal; use windmill_common::jobs::JobPayload; use windmill_common::jobs::JobTriggerKind; use windmill_common::jobs::OnBehalfOf; @@ -511,14 +510,15 @@ pub async fn push_scheduled_job<'c>( if let Some(tag) = tag.as_deref().filter(|t| !t.is_empty()) { let is_super_admin = windmill_common::auth::is_super_admin_email(db, &email).await?; - check_tag_available_for_workspace_internal( + crate::check_tag_available_for_push( db, &schedule.workspace_id, &tag, + &crate::PushArgs::from(&args), is_super_admin, None, // no token for schedules so no scopes so no scope_tags ) - .warn_after_seconds_with_sql(1, "check_tag_available_for_workspace_internal".to_string()) + .warn_after_seconds_with_sql(1, "check_tag_available_for_push".to_string()) .await?; } diff --git a/backend/windmill-runtime-nativets/src/windmill-client.js b/backend/windmill-runtime-nativets/src/windmill-client.js index 7c7d649042..c2dbd1fef1 100644 --- a/backend/windmill-runtime-nativets/src/windmill-client.js +++ b/backend/windmill-runtime-nativets/src/windmill-client.js @@ -6457,7 +6457,7 @@ var AppService = class { * @param data.workspace * @param data.path * @param data.requestBody update app - * @returns string app updated + * @returns AppDeployed the path the app now lives at and the version this deploy wrote * @throws ApiError */ static updateApp(data) { diff --git a/backend/windmill-trigger/src/trigger_helpers.rs b/backend/windmill-trigger/src/trigger_helpers.rs index 6a37bd0da1..7e47613d1a 100644 --- a/backend/windmill-trigger/src/trigger_helpers.rs +++ b/backend/windmill-trigger/src/trigger_helpers.rs @@ -880,7 +880,8 @@ async fn trigger_script_with_retry_and_error_handler<'c>( delete_after_secs, ); - check_tag_available_for_workspace(&db, &workspace_id, &tag, &authed).await?; + let push_args = PushArgs { args: &args.args, extra: args.extra }; + check_tag_available_for_workspace(&db, &workspace_id, &tag, &push_args, &authed).await?; let return_tx = tx_o.is_some(); @@ -907,8 +908,6 @@ async fn trigger_script_with_retry_and_error_handler<'c>( ) }; - let push_args = PushArgs { args: &args.args, extra: args.extra }; - let retryable_job_payload = match job_payload { JobPayload::ScriptHash { hash, diff --git a/backend/windmill-types/src/scripts.rs b/backend/windmill-types/src/scripts.rs index 9a059da6e8..9e29731505 100644 --- a/backend/windmill-types/src/scripts.rs +++ b/backend/windmill-types/src/scripts.rs @@ -507,6 +507,10 @@ pub struct ScriptHistory { pub deployment_msg: Option, #[serde(skip_serializing_if = "Option::is_none")] pub created_at: Option>, + /// Who deployed this version — the version picker names it, so a reader can + /// tell their own deploys from a teammate's. + #[serde(skip_serializing_if = "Option::is_none")] + pub created_by: Option, } #[derive(Deserialize)] diff --git a/backend/windmill-worker/src/bun_executor.rs b/backend/windmill-worker/src/bun_executor.rs index 117fbda086..841ea8861a 100644 --- a/backend/windmill-worker/src/bun_executor.rs +++ b/backend/windmill-worker/src/bun_executor.rs @@ -2689,13 +2689,13 @@ try {{ /// Resolve a module file from the parent script's modules map. /// For Script jobs, fetches from the `script` table by hash. /// For Preview jobs, fetches from `v2_job.raw_code` (modules stored inline). -fn resolve_parent_module( - modules: &Option>, +fn resolve_parent_module<'a>( + modules: &'a Option>, module_key: &str, -) -> error::Result { +) -> error::Result<&'a windmill_common::scripts::ScriptModule> { if let Some(modules) = modules { if let Some(module) = modules.get(module_key) { - return Ok(module.clone()); + return Ok(module); } } Err(error::Error::ExecutionErr(format!( @@ -2716,7 +2716,7 @@ pub async fn handle_wac_v2_output( ) -> error::Result> { use crate::wac_executor::{ load_checkpoint, parse_wac_output, update_checkpoint_for_dispatch, - wac_cancelled_mid_segment, WacOutput, WacPark, + wac_cancelled_mid_segment, WacOutput, WacPark, WacStepDispatch, }; use serde_json::Value; use windmill_common::get_latest_flow_version_info_for_path; @@ -2724,7 +2724,7 @@ pub async fn handle_wac_v2_output( use windmill_common::runnable_settings::{ ConcurrencySettings, ConcurrencySettingsWithCustom, DebouncingSettings, }; - use windmill_queue::{push, PushArgs, PushIsolationLevel}; + use windmill_queue::{check_tag_available_for_push, push, PushArgs, PushIsolationLevel}; let output = parse_wac_output(&result)?; @@ -2950,6 +2950,129 @@ pub async fn handle_wac_v2_output( ))), }?; + // Resolve what every task runs, and check any tag it picks, before the parent parks: + // a task that cannot be pushed then fails the dispatch with no sibling queued. The + // parent's code and input are built per child at push time, so a fan-out never holds + // a copy of them for each child at once. + enum ChildRunnable<'a> { + // Re-runs the parent with `_executing_key`. + Parent, + // A `./` module of the parent script. + Module(&'a windmill_common::scripts::ScriptModule), + // A deployed script or flow. + Deployed(JobPayload), + } + struct ResolvedChild<'a> { + runnable: ChildRunnable<'a>, + email: String, + permissioned_as: String, + } + // The input a task with a runnable of its own is pushed with; the parent re-run gets + // the parent's. Both the tag check and the push build it here, so the check resolves + // `$args[...]` against what the child actually receives. + let own_args = |step: &WacStepDispatch| -> HashMap> { + step.args + .iter() + .map(|(k, v)| { + let raw = serde_json::value::to_raw_value(v).unwrap(); + (k.clone(), raw) + }) + .collect() + }; + let mut children: Vec = Vec::with_capacity(num_steps); + for step in &steps { + let (runnable, on_behalf_of) = match step.dispatch_type.as_str() { + "script" if step.script.starts_with("./") => { + // Module-relative path: resolve from parent script's modules + let module_key = step.script.strip_prefix("./").unwrap(); + let module = resolve_parent_module(modules, module_key)?; + // Inline module code, not a separate runnable: it has no + // identity of its own and runs as the parent. + (ChildRunnable::Module(module), None) + } + "script" => { + // Resolve script path to job payload (handles hash, lang, etc.) + let (payload, _, _, _, _, on_behalf_of) = script_path_to_payload( + &step.script, + None, // no authed db for background workers + db.clone(), + &job.workspace_id, + Some(true), // skip preprocessor + ) + .await?; + (ChildRunnable::Deployed(payload), on_behalf_of) + } + "flow" => { + let flow_info = get_latest_flow_version_info_for_path( + None, + db, + &job.workspace_id, + &step.script, + true, + ) + .await?; + let payload = JobPayload::Flow { + path: step.script.clone(), + dedicated_worker: flow_info.dedicated_worker, + apply_preprocessor: false, + version: flow_info.version, + labels: flow_info.labels.clone(), + }; + let on_behalf_of = flow_info.on_behalf_of(&job.workspace_id, db).await?; + (ChildRunnable::Deployed(payload), on_behalf_of) + } + // "inline" — re-run parent with _executing_key + _ => (ChildRunnable::Parent, None), + }; + + // A target runnable that opts into on-behalf-of runs under its own + // identity, never the caller's, so a step that reaches it through a + // workflow cannot widen or narrow its permissions. `created_by` still + // credits the caller, matching how the run API pushes these jobs. + let (email, permissioned_as) = match on_behalf_of { + Some(on_behalf_of) => (on_behalf_of.email, on_behalf_of.permissioned_as), + None => ( + job.permissioned_as_email.clone(), + job.permissioned_as.clone(), + ), + }; + + // A task inheriting the parent's tag skips the check: that tag was + // checked when the parent was pushed, and a dedicated worker's tag is + // never in CUSTOM_TAGS. + if let Some(tag) = step + .tag + .as_deref() + .filter(|t| !t.is_empty() && *t != job.tag.as_str()) + { + let is_super_admin = + windmill_common::auth::is_super_admin_email(db, &email).await?; + let step_args = + (!matches!(runnable, ChildRunnable::Parent)).then(|| own_args(step)); + let args = + PushArgs { args: step_args.as_ref().unwrap_or(&parent_args), extra: None }; + check_tag_available_for_push( + db, + &job.workspace_id, + tag, + &args, + is_super_admin, + None, + ) + .warn_after_seconds_with_sql(1, "check_tag_available_for_push".to_string()) + .await + .map_err(|e| match e { + error::Error::BadRequest(msg) => error::Error::BadRequest(format!( + "task '{}' cannot run on tag '{tag}': {msg}", + step.name + )), + e => e, + })?; + } + + children.push(ResolvedChild { runnable, email, permissioned_as }); + } + // Step 1: Save checkpoint, suspend parent, and seed child checkpoints // in a single transaction — all BEFORE children become visible. let segment_ms; @@ -3034,106 +3157,37 @@ pub async fn handle_wac_v2_output( // partial failure (e.g. pushing child 3 of 5 fails). let mut pushed_ids: Vec = Vec::with_capacity(num_steps); let push_result: error::Result<()> = async { - for (step, (_, child_uuid)) in steps.iter().zip(job_ids.iter()) { + for ((step, (_, child_uuid)), child) in + steps.iter().zip(job_ids.iter()).zip(children) + { // A task with a runnable of its own (a deployed script or flow) queues // at that runnable's priority; any other task is the parent's code and // queues at the parent's. let own_runnable = matches!(step.dispatch_type.as_str(), "script" | "flow") && !step.script.starts_with("./"); - // Resolve job payload based on dispatch_type - let (job_payload, child_args, is_external, on_behalf_of) = - match step.dispatch_type.as_str() { - "script" if step.script.starts_with("./") => { - // Module-relative path: resolve from parent script's modules - let module_key = step.script.strip_prefix("./").unwrap(); - let module = resolve_parent_module(modules, module_key)?; - let payload = JobPayload::Code(RawCode { - content: module.content, - path: job.runnable_path.clone(), - hash: None, - language: module.language, - lock: module.lock, - cache_ttl: None, - cache_ignore_s3_path: None, - dedicated_worker: None, - concurrency_settings: ConcurrencySettingsWithCustom::default(), - debouncing_settings: DebouncingSettings::default(), - modules: None, - tag: None, - }); - let step_args: HashMap> = step - .args - .iter() - .map(|(k, v)| { - let raw = serde_json::value::to_raw_value(v).unwrap(); - (k.clone(), raw) - }) - .collect(); - // Inline module code, not a separate runnable: it has no - // identity of its own and runs as the parent. - (payload, step_args, true, None) - } - "script" => { - // Resolve script path to job payload (handles hash, lang, etc.) - let (payload, _, _, _, _, on_behalf_of) = script_path_to_payload( - &step.script, - None, // no authed db for background workers - db.clone(), - &job.workspace_id, - Some(true), // skip preprocessor - ) - .await?; - let step_args: HashMap> = step - .args - .iter() - .map(|(k, v)| { - let raw = serde_json::value::to_raw_value(v).unwrap(); - (k.clone(), raw) - }) - .collect(); - (payload, step_args, true, on_behalf_of) - } - "flow" => { - let flow_info = get_latest_flow_version_info_for_path( - None, - db, - &job.workspace_id, - &step.script, - true, - ) - .await?; - let payload = JobPayload::Flow { - path: step.script.clone(), - dedicated_worker: flow_info.dedicated_worker, - apply_preprocessor: false, - version: flow_info.version, - labels: flow_info.labels.clone(), - }; - let on_behalf_of = - flow_info.on_behalf_of(&job.workspace_id, db).await?; - let step_args: HashMap> = step - .args - .iter() - .map(|(k, v)| { - let raw = serde_json::value::to_raw_value(v).unwrap(); - (k.clone(), raw) - }) - .collect(); - (payload, step_args, true, on_behalf_of) - } - _ => { - // "inline" — re-run parent with _executing_key - ( - job_payload_template.clone(), - parent_args.clone(), - false, - None, - ) - } - }; - - let push_args = PushArgs { args: &child_args, extra: None }; + let is_external = !matches!(child.runnable, ChildRunnable::Parent); + let job_payload = match child.runnable { + ChildRunnable::Parent => job_payload_template.clone(), + ChildRunnable::Module(module) => JobPayload::Code(RawCode { + content: module.content.clone(), + path: job.runnable_path.clone(), + hash: None, + language: module.language, + lock: module.lock.clone(), + cache_ttl: None, + cache_ignore_s3_path: None, + dedicated_worker: None, + concurrency_settings: ConcurrencySettingsWithCustom::default(), + debouncing_settings: DebouncingSettings::default(), + modules: None, + tag: None, + }), + ChildRunnable::Deployed(payload) => payload, + }; + let step_args = is_external.then(|| own_args(step)); + let push_args = + PushArgs { args: step_args.as_ref().unwrap_or(&parent_args), extra: None }; // Apply step-level overrides to payload (cache, concurrency) let mut job_payload = job_payload; @@ -3181,21 +3235,6 @@ pub async fn handle_wac_v2_output( } } - // A target runnable that opts into on-behalf-of runs under its own - // identity, never the caller's, so a step that reaches it through a - // workflow cannot widen or narrow its permissions. `created_by` still - // credits the caller, matching how the run API pushes these jobs. - let (child_email, child_permissioned_as) = match on_behalf_of.as_ref() { - Some(on_behalf_of) => ( - on_behalf_of.email.as_str(), - on_behalf_of.permissioned_as.clone(), - ), - None => ( - job.permissioned_as_email.as_str(), - job.permissioned_as.clone(), - ), - }; - let (_, mut tx) = push( db, PushIsolationLevel::IsolatedRoot(db.clone()), @@ -3203,8 +3242,8 @@ pub async fn handle_wac_v2_output( job_payload, push_args, &job.created_by, - child_email, - child_permissioned_as, + &child.email, + child.permissioned_as, None, None, None, diff --git a/backend/windmill-worker/src/worker_flow.rs b/backend/windmill-worker/src/worker_flow.rs index dbdba3e6ed..0ab4e2871c 100644 --- a/backend/windmill-worker/src/worker_flow.rs +++ b/backend/windmill-worker/src/worker_flow.rs @@ -44,8 +44,7 @@ use windmill_common::flow_status::{ }; use windmill_common::flows::{add_virtual_items_if_necessary, Branch, FlowNodeId, StopAfterIf}; use windmill_common::jobs::{ - check_tag_available_for_workspace_internal, script_path_to_payload, JobKind, JobPayload, - OnBehalfOf, RawCode, ENTRYPOINT_OVERRIDE, + script_path_to_payload, JobKind, JobPayload, OnBehalfOf, RawCode, ENTRYPOINT_OVERRIDE, }; use windmill_common::runnable_settings::{ ConcurrencySettingsWithCustom, DebouncingSettings, RunnableSettingsTrait, @@ -68,11 +67,12 @@ use windmill_common::{ }; use windmill_queue::schedule::get_schedule_opt; use windmill_queue::{ - add_completed_job, add_completed_job_error, append_logs, get_mini_pulled_job, - insert_concurrency_key_capped, interpolate_args, render_tag_path, - report_error_to_workspace_handler_or_critical_side_channel, tag_reads_args, - tag_reads_flow_expr, try_schedule_next_job, CanceledBy, FlowRunners, MiniCompletedJob, - MiniPulledJob, PushArgs, PushIsolationLevel, SameWorkerPayload, WrappedError, RE_FLOW_EXPR_TAG, + add_completed_job, add_completed_job_error, append_logs, + check_tag_written_as_available_for_push, get_mini_pulled_job, insert_concurrency_key_capped, + render_tag_path, report_error_to_workspace_handler_or_critical_side_channel, resolve_push_tag, + tag_reads_args, tag_reads_flow_expr, try_schedule_next_job, CanceledBy, FlowRunners, + MiniCompletedJob, MiniPulledJob, PushArgs, PushIsolationLevel, SameWorkerPayload, WrappedError, + RE_FLOW_EXPR_TAG, }; use windmill_audit::audit_oss::audit_log; @@ -1520,15 +1520,6 @@ pub async fn update_flow_status_after_job_completion_internal( .is_some_and(|ck| ck.contains("$args")) }); let require_args = concurrency_requires_args || has_debouncing; - let mut tag = tag_and_concurrency_key.as_ref().and_then(|x| x.tag.clone()); - // `$workspace` does not depend on the preprocessor's output, so it has to resolve even - // when nothing forced us to fetch args. Leaving it to the `$args` branch below writes a - // `$workspace`-only tag back verbatim, naming a queue no worker serves. - if let Some(t) = tag.as_ref().filter(|t| t.contains("$workspace")) { - let tag_ws = - windmill_queue::tags::tag_workspace_id(&flow_job.workspace_id, db).await; - tag = Some(t.replace("$workspace", &tag_ws)); - } let concurrency_key = tag_and_concurrency_key .as_ref() .and_then(|x| x.concurrency_key.clone()); @@ -1576,10 +1567,6 @@ pub async fn update_flow_status_after_job_completion_internal( ) .await?; } - if let Some(t) = tag { - // `$workspace` is already resolved above; this fills in `$args`. - tag = Some(interpolate_args(t, &args, &flow_job.workspace_id)); - } } else if concurrent_limit.is_some() { insert_concurrency_key_capped( &flow_job.workspace_id, @@ -1594,6 +1581,23 @@ pub async fn update_flow_status_after_job_completion_internal( .await?; } + // Resolved as `push` resolves a tag, so a `$flow_expr[...]` or empty one comes out as + // `None` and the job keeps its current tag instead of queueing on the literal text. + let tag = match tag_and_concurrency_key + .as_ref() + .and_then(|x| x.tag.as_deref()) + { + // `run_flow` checked this tag for whoever ran the flow, an `$args[...]` in it as + // written since only this point knows its values: an entry admitting it as + // written admits every value here. + Some(t) => { + let no_args = HashMap::new(); + let args = PushArgs::from(fetched_args.as_ref().unwrap_or(&no_args)); + resolve_push_tag(t, &args, &flow_job.workspace_id, db).await + } + None => None, + }; + let scheduled_for: Option> = { #[cfg(feature = "private")] { @@ -4494,33 +4498,7 @@ async fn push_next_flow_job( ) }; - // Check tag availability for flow substeps to prevent abuse. - // Skip when the substep inherits the parent flow's tag: that tag was already - // validated at flow push time, and for dedicated flows it is the auto-generated - // `{workspace_id}:flow/{path}` tag which is never in CUSTOM_TAGS. - if let Some(tag_str) = tag - .as_deref() - .filter(|t| !t.is_empty() && *t != flow_job.tag.as_str()) - { - // A step with its own on-behalf-of carries a cached dispatch address, up to one - // notify poll stale; accepted, see `get_email_from_permissioned_as`. - let is_super_admin = windmill_common::auth::is_super_admin_email(db, email).await?; - check_tag_available_for_workspace_internal( - db, - &flow_job.workspace_id, - tag_str, - is_super_admin, - None, // no token for flow substeps so no scopes so no scope_tags - ) - .warn_after_seconds_with_sql( - 1, - "check_tag_available_for_workspace_internal".to_string(), - ) - .await?; - } - - // Resolved only after the check: CUSTOM_TAGS allows the template, so its value may name - // any queue, as the value of an `$args[...]` tag does. + let written_tag = tag.clone(); let mut tag_err = None; let tag = match tag { Some(t) if err.is_none() && tag_reads_flow_expr(&t) => { @@ -4543,6 +4521,31 @@ async fn push_next_flow_job( }; let err = err.or(tag_err.as_ref()); + // Check tag availability for flow substeps to prevent abuse. It runs on the tag with its + // `$flow_expr[...]` filled in, as `push` fills in the rest, since that is what picks the + // queue. Skip when the substep inherits the parent flow's tag: that tag was already + // validated at flow push time, and for dedicated flows it is the auto-generated + // `{workspace_id}:flow/{path}` tag which is never in CUSTOM_TAGS. + if let Some(tag_str) = tag + .as_deref() + .filter(|t| !t.is_empty() && *t != flow_job.tag.as_str()) + { + // A step with its own on-behalf-of carries a cached dispatch address, up to one + // notify poll stale; accepted, see `get_email_from_permissioned_as`. + let is_super_admin = windmill_common::auth::is_super_admin_email(db, email).await?; + check_tag_written_as_available_for_push( + db, + &flow_job.workspace_id, + written_tag.as_deref().unwrap_or(tag_str), + tag_str, + &push_args, + is_super_admin, + None, // no token for flow substeps so no scopes so no scope_tags + ) + .warn_after_seconds_with_sql(1, "check_tag_available_for_push".to_string()) + .await?; + } + let evaluated_timeout = if let Some(timeout_transform) = &module.timeout { let ctx = get_transform_context(&flow_job, &previous_id, &status); diff --git a/benchmarks/lib.ts b/benchmarks/lib.ts index ed07c4c140..1e6cfdd474 100644 --- a/benchmarks/lib.ts +++ b/benchmarks/lib.ts @@ -2,7 +2,7 @@ import { sleep } from "https://deno.land/x/sleep@v1.2.1/mod.ts"; import * as windmill from "https://deno.land/x/windmill@v1.174.0/mod.ts"; import * as api from "https://deno.land/x/windmill@v1.174.0/windmill-api/index.ts"; -export const VERSION = "v1.815.0"; +export const VERSION = "v1.817.0"; export async function login(email: string, password: string): Promise { return await windmill.UserService.login({ diff --git a/chat-sdk/package-lock.json b/chat-sdk/package-lock.json index 36407f964f..ff470d4ffe 100644 --- a/chat-sdk/package-lock.json +++ b/chat-sdk/package-lock.json @@ -1,12 +1,12 @@ { "name": "windmill-chat", - "version": "1.815.0", + "version": "1.817.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "windmill-chat", - "version": "1.815.0", + "version": "1.817.0", "license": "Apache-2.0", "devDependencies": { "@ai-sdk/react": "^4.0.102", diff --git a/chat-sdk/package.json b/chat-sdk/package.json index 2c42ca3565..5655adc64e 100644 --- a/chat-sdk/package.json +++ b/chat-sdk/package.json @@ -1,7 +1,7 @@ { "name": "windmill-chat", "description": "Build chat interfaces on Windmill flows deployed in chat mode, from any frontend or raw app", - "version": "1.815.0", + "version": "1.817.0", "author": "Ruben Fiszel", "license": "Apache-2.0", "homepage": "https://github.com/windmill-labs/windmill/tree/main/chat-sdk#readme", diff --git a/cli/src/commands/app/raw_apps.ts b/cli/src/commands/app/raw_apps.ts index 8d82edb9df..36c8260e25 100644 --- a/cli/src/commands/app/raw_apps.ts +++ b/cli/src/commands/app/raw_apps.ts @@ -55,6 +55,8 @@ export interface AppFile { tables?: string[]; datatable?: string; schema?: string; + /** The role the app uses each data table through, by data table name. */ + roles?: Record; }; // Mirrors granular ACLs on the raw_app path. Synced via /acls/* by // applyExtraPermsDiff — never through update_app_raw — so a perm-only diff --git a/cli/src/commands/sync/sync.ts b/cli/src/commands/sync/sync.ts index 6040afc747..6185decc5a 100644 --- a/cli/src/commands/sync/sync.ts +++ b/cli/src/commands/sync/sync.ts @@ -1153,7 +1153,7 @@ export function ZipFSElement( ? "script" : p.endsWith(".resource.json") ? "resource" - : p.startsWith("dependencies/") + : isWorkspaceDependencies(p) ? "dependencies" : "other"; @@ -2752,7 +2752,7 @@ export async function compareDynFSElement( if (skipMetadata) { continue; } - if (k.startsWith("dependencies/")) { + if (isWorkspaceDependencies(k)) { if (!workspaceDependenciesPathToLanguageAndFilename(k)) { log.warn(`Skipping unrecognized workspace dependencies file: ${k}`); continue; @@ -3594,12 +3594,15 @@ export async function pull( ) { if ((opts as any).jsonOutput) log.setSilent(true); const originalCliOpts = { ...opts }; - opts = await mergeConfigWithConfigFile(opts); - - // --include-secrets overrides skipSecrets from wmill.yaml - if ((originalCliOpts as any).includeSecrets) { - opts.skipSecrets = false; - } + const withConfigFile = async () => { + const merged = await mergeConfigWithConfigFile({ ...originalCliOpts }); + // --include-secrets overrides skipSecrets from wmill.yaml + if ((originalCliOpts as any).includeSecrets) { + merged.skipSecrets = false; + } + return merged; + }; + opts = await withConfigFile(); // Resolve workspace name for config lookups. // --branch resolves git branch → workspace name (deprecated but still supported). @@ -3634,10 +3637,6 @@ export async function pull( throw error; } - if (opts.stateful) { - await mkdir(path.join(process.cwd(), ".wmill"), { recursive: true }); - } - const workspace = await resolveWorkspace(opts, wsNameForConfig); await requireLogin(opts); @@ -3729,6 +3728,14 @@ export async function pull( }); return; } + + // The pull writes into the branch now checked out, so its wmill.yaml + // applies, not the cloned branch's: a fork branch that turned on + // `dedupeLockfiles` would otherwise get one lockfile per script back. + if (getCurrentGitBranch() !== clonedBranchName) { + opts = await withConfigFile(); + wsNameForConfig = resolveWsNameForConfigFromFlags(opts); + } } // If wsNameForConfig wasn't set from flags, infer from the resolved profile @@ -3763,6 +3770,10 @@ export async function pull( // Merge CLI flags with resolved settings (CLI flags take precedence only for explicit overrides) opts = mergeCliWithEffectiveOptions(originalCliOpts, effectiveOpts); + if (opts.stateful) { + await mkdir(path.join(process.cwd(), ".wmill"), { recursive: true }); + } + const codebases = await listSyncCodebases(opts); log.info( diff --git a/cli/src/commands/workspace/fork.ts b/cli/src/commands/workspace/fork.ts index 219252bb7a..0aca0984ff 100644 --- a/cli/src/commands/workspace/fork.ts +++ b/cli/src/commands/workspace/fork.ts @@ -239,6 +239,7 @@ async function createWorkspaceFork( interface ForkedDatatableInfo { name: string; new_dbname: string; + fork_behavior?: "schema_only" | "schema_and_data"; } const forkedDatatables: ForkedDatatableInfo[] = []; @@ -285,6 +286,23 @@ async function createWorkspaceFork( const newDbName = `${trueWorkspaceId.replace(/-/g, "_")}__${dt.name}`; + const forkBehavior = dtBehavior as "schema_only" | "schema_and_data"; + // A server that reports `permissioned` copies each data table in the fork request itself, + // and refuses a database copied beforehand. An older one only takes a database copied here. + if (typeof dt.permissioned === "boolean") { + log.info( + colors.blue( + ` Datatable "${dt.name}" will be cloned (${forkBehavior === "schema_only" ? "schema" : "schema + data"}) into "${newDbName}" when the fork is created.` + ) + ); + forkedDatatables.push({ + name: dt.name, + new_dbname: newDbName, + fork_behavior: forkBehavior, + }); + continue; + } + try { log.info( colors.blue(` Creating database "${newDbName}" for datatable "${dt.name}"...`) @@ -300,7 +318,7 @@ async function createWorkspaceFork( log.info( colors.blue( - ` Importing ${dtBehavior === "schema_only" ? "schema" : "schema + data"}...` + ` Importing ${forkBehavior === "schema_only" ? "schema" : "schema + data"}...` ) ); @@ -310,7 +328,7 @@ async function createWorkspaceFork( source: `datatable://${dt.name}`, target: `datatable://${dt.name}`, target_dbname_override: newDbName, - fork_behavior: dtBehavior as "schema_only" | "schema_and_data", + fork_behavior: forkBehavior, }, }); @@ -336,6 +354,8 @@ async function createWorkspaceFork( id: trueWorkspaceId, name: opts.createWorkspaceName ?? workspaceName ?? trueWorkspaceId, color: forkColor, + // So a clone the fork would refuse is refused before any branch is created. + forked_datatables: forkedDatatables, }, }); if (gitSyncJobIds && gitSyncJobIds.length > 0) { diff --git a/cli/src/core/constants.ts b/cli/src/core/constants.ts index ab1bcf0a87..9728b6edd7 100644 --- a/cli/src/core/constants.ts +++ b/cli/src/core/constants.ts @@ -10,4 +10,4 @@ export const WM_FORK_PREFIX = "wm-fork"; // (e.g. utils.ts) can read it without importing main.ts and creating a circular // dependency (main → workspace → utils → main) that triggers a TDZ. // Re-exported from main.ts for backwards compatibility. -export const VERSION = "1.815.0"; +export const VERSION = "1.817.0"; diff --git a/cli/src/guidance/skills.gen.ts b/cli/src/guidance/skills.gen.ts index 6da10b2504..81895997b9 100644 --- a/cli/src/guidance/skills.gen.ts +++ b/cli/src/guidance/skills.gen.ts @@ -5826,6 +5826,8 @@ data: tables: - main/users # Table in public schema - main/app_schema:items # Table in specific schema + roles: # Optional: the role the app uses each datatable through + main: analyst \`\`\` **Table reference formats:** @@ -5833,6 +5835,8 @@ data: - \`/\` — Specific table in public schema - \`/:
\` — Table in specific schema +**Roles:** when a datatable is under roles, its queries run as a role, which only reaches what it was granted. \`roles\` records the role the app uses each datatable through; the app's code must pass the same role: \`wmill.datatable('main', { role: 'analyst' })\` in TypeScript, \`wmill.datatable('main', role='analyst')\` in Python. A datatable without an entry is used as its default role. + ## SQL Migrations (sql_to_apply/) The \`sql_to_apply/\` folder is for creating/modifying database tables during development. diff --git a/cli/src/types.ts b/cli/src/types.ts index 5713d06390..866b2be21e 100644 --- a/cli/src/types.ts +++ b/cli/src/types.ts @@ -391,7 +391,7 @@ export function getTypeStrFromPath( if (isSharedLockPath(p)) { return "shared_lock"; } - if (p.startsWith("dependencies" + SEP)) { + if (isWorkspaceDependencies(p)) { return "workspace_dependencies"; } if (isFileResource(p) || isFilesetResource(p)) { diff --git a/cli/src/utils/script_common.ts b/cli/src/utils/script_common.ts index d804318491..611268c27c 100644 --- a/cli/src/utils/script_common.ts +++ b/cli/src/utils/script_common.ts @@ -45,7 +45,9 @@ export const workspaceDependenciesLanguages: WorkspaceDependenciesLanguage[] = [ ] as const; export function workspaceDependenciesPathToLanguageAndFilename(path: string): { name: string | undefined, language: ScriptLanguage } | undefined { - const relativePath = path.replace("dependencies/", ""); + // Sync paths carry the platform separator, while a set's name is always + // forward-slashed. + const relativePath = path.replaceAll("\\", "/").replace("dependencies/", ""); for (const { filename, language } of workspaceDependenciesLanguages) { if (relativePath.endsWith(filename)) { return { diff --git a/cli/src/utils/utils.ts b/cli/src/utils/utils.ts index 0801c03911..526dffbde4 100644 --- a/cli/src/utils/utils.ts +++ b/cli/src/utils/utils.ts @@ -223,8 +223,9 @@ export function isRawAppFile(path: string): boolean { return isRawAppPath(path); } +/** Sync paths carry the platform separator: `dependencies\x` on Windows. */ export function isWorkspaceDependencies(path: string): boolean { - return path.startsWith("dependencies/"); + return path.startsWith("dependencies/") || path.startsWith("dependencies\\"); } export function printSync(input: string | Uint8Array) { diff --git a/cli/test/gitsync_deploy_branch_config.test.ts b/cli/test/gitsync_deploy_branch_config.test.ts new file mode 100644 index 0000000000..eeab7a0120 --- /dev/null +++ b/cli/test/gitsync_deploy_branch_config.test.ts @@ -0,0 +1,94 @@ +import { expect, test } from "bun:test"; +import { execFileSync } from "node:child_process"; +import { existsSync } from "node:fs"; +import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { createRemoteWorkspaceDeps, withTestBackend } from "./test_backend.ts"; + +function git(cwd: string, ...args: string[]): string { + return execFileSync("git", args, { cwd, encoding: "utf8" }).trim(); +} + +// The git-sync deploy callback starts in a clone of the tracked branch and +// switches to the fork's branch before pulling. What it writes there must follow +// that branch's wmill.yaml: a fork branch that turned on `dedupeLockfiles` keeps +// its shared lockfile instead of getting a `.script.lock` per script back. +test("git-sync fork deploy follows the fork branch's wmill.yaml", async () => { + await withTestBackend(async (backend) => { + const post = (path: string, body: unknown) => + backend.apiRequest!(`/api/w/${backend.workspace}${path}`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(body), + }); + await createRemoteWorkspaceDeps(backend, "python3", "wmill\n"); + await post("/folders/create", { name: "dedupe" }); + const script = await post("/scripts/create", { + path: "f/dedupe/a", + summary: "", + description: "", + content: "def main():\n pass\n", + language: "python3", + lock: "wmill==1.0.0\n", + }); + expect(script.ok).toBe(true); + + const bare = await mkdtemp(join(tmpdir(), "wmill_deploy_cfg_bare_")); + const seed = await mkdtemp(join(tmpdir(), "wmill_deploy_cfg_seed_")); + const work = await mkdtemp(join(tmpdir(), "wmill_deploy_cfg_work_")); + try { + execFileSync("git", ["init", "--bare", "--initial-branch=main", bare]); + git(seed, "init", "--initial-branch=main"); + git(seed, "config", "user.email", "seed@windmill.dev"); + git(seed, "config", "user.name", "seed"); + git(seed, "remote", "add", "origin", `file://${bare}`); + const wmillYaml = "defaultTs: bun\nincludes:\n - f/dedupe/**\nexcludes: []\n"; + await writeFile(join(seed, "wmill.yaml"), wmillYaml); + git(seed, "add", "-A"); + git(seed, "commit", "-m", "main"); + git(seed, "push", "origin", "main"); + + // Any parent id makes the callback deploy as a fork, to this branch. + const forkBranch = `wm-fork/main/${backend.workspace}`; + git(seed, "checkout", "-b", forkBranch); + await writeFile( + join(seed, "wmill.yaml"), + wmillYaml + "dedupeLockfiles: true\n", + ); + const pulled = await backend.runCLICommand(["sync", "pull", "--yes"], seed); + expect(pulled.code).toBe(0); + expect(existsSync(join(seed, "locks/requirements.in.lock"))).toBe(true); + git(seed, "add", "-A"); + git(seed, "commit", "-m", "dedupe"); + git(seed, "push", "origin", forkBranch); + + // Git for Windows defaults to autocrlf=true, which checks the pulled LF + // files out as CRLF; the deploy's pull would rewrite every one of them + // and the clean-tree assertion below would read that as a change. + git(work, "clone", "-c", "core.autocrlf=false", `file://${bare}`, "."); + const deployed = await backend.runCLICommand( + [ + "sync", + "git-deploy", + "--repository", + "u/test/unused", + "--git-deploy-items", + JSON.stringify([ + { path_type: "script", path: "f/dedupe/a", commit_msg: "deploy" }, + ]), + "--parent-workspace-id", + "parent", + ], + work, + ); + expect(deployed.code).toBe(0); + expect(git(work, "rev-parse", "--abbrev-ref", "HEAD")).toBe(forkBranch); + expect(git(work, "status", "--porcelain")).toBe(""); + } finally { + await rm(bare, { recursive: true, force: true }); + await rm(seed, { recursive: true, force: true }); + await rm(work, { recursive: true, force: true }); + } + }); +}); diff --git a/cli/test/sync_pull_push.test.ts b/cli/test/sync_pull_push.test.ts index 24c4a282a3..fa52510e6c 100644 --- a/cli/test/sync_pull_push.test.ts +++ b/cli/test/sync_pull_push.test.ts @@ -2185,18 +2185,20 @@ describe("settings sync", () => { const files = await listFilesRecursive(tempDir); expect(files).toContain("settings.yaml"); - // Read and modify a safe setting (webhook URL) + // Read and modify a safe setting (webhook URL). The server resolves the + // webhook host when it is saved and refuses one that does not resolve to a + // public address, so this must be a real public hostname. + const webhook = "https://example.com/hook"; const settingsContent = await readFile(`${tempDir}/settings.yaml`, "utf-8"); let modifiedSettings: string; if (settingsContent.includes("webhook:")) { modifiedSettings = settingsContent.replace( /webhook:.*/, - 'webhook: "https://test-webhook.example.com/hook"' + `webhook: "${webhook}"` ); } else { - modifiedSettings = - settingsContent + '\nwebhook: "https://test-webhook.example.com/hook"\n'; + modifiedSettings = settingsContent + `\nwebhook: "${webhook}"\n`; } await writeFile(`${tempDir}/settings.yaml`, modifiedSettings, "utf-8"); @@ -2213,7 +2215,7 @@ describe("settings sync", () => { ); expect(apiResp.status).toEqual(200); const settingsData = await apiResp.json(); - expect(settingsData.webhook).toEqual("https://test-webhook.example.com/hook"); + expect(settingsData.webhook).toEqual(webhook); }); }); }); diff --git a/cli/test/workspace_deps_path_unit.test.ts b/cli/test/workspace_deps_path_unit.test.ts new file mode 100644 index 0000000000..45d725c6e5 --- /dev/null +++ b/cli/test/workspace_deps_path_unit.test.ts @@ -0,0 +1,23 @@ +import { expect, test } from "bun:test"; +import { isWorkspaceDependencies } from "../src/utils/utils.ts"; +import { workspaceDependenciesPathToLanguageAndFilename } from "../src/utils/script_common.ts"; + +// Sync paths carry the platform separator. A Windows path parsed to a different +// set name pushes the file to a set the workspace does not have. +test("workspace dependency paths parse the same with either separator", () => { + for (const sep of ["/", "\\"]) { + expect(isWorkspaceDependencies(`dependencies${sep}requirements.in`)).toBe(true); + expect( + workspaceDependenciesPathToLanguageAndFilename(`dependencies${sep}requirements.in`), + ).toEqual({ name: undefined, language: "python3" }); + expect( + workspaceDependenciesPathToLanguageAndFilename(`dependencies${sep}team.package.json`), + ).toEqual({ name: "team", language: "bun" }); + expect( + workspaceDependenciesPathToLanguageAndFilename( + `dependencies${sep}team${sep}python.requirements.in`, + ), + ).toEqual({ name: "team/python", language: "python3" }); + } + expect(isWorkspaceDependencies("f\\dependencies\\requirements.in")).toBe(false); +}); diff --git a/docker/DockerfileCuda b/docker/DockerfileCuda index 098670f9a3..031921070b 100644 --- a/docker/DockerfileCuda +++ b/docker/DockerfileCuda @@ -1,4 +1,5 @@ -FROM ghcr.io/windmill-labs/windmill-ee:dev +ARG BASE_IMAGE=ghcr.io/windmill-labs/windmill-ee:dev +FROM ${BASE_IMAGE} # Install any necessary dependencies for managing repositories and packages RUN apt-get update && apt-get install -y curl gnupg2 diff --git a/docker/DockerfileFull b/docker/DockerfileFull index b827208c1d..8ef6be78bb 100644 --- a/docker/DockerfileFull +++ b/docker/DockerfileFull @@ -1,4 +1,5 @@ -FROM ghcr.io/windmill-labs/windmill:dev +ARG BASE_IMAGE=ghcr.io/windmill-labs/windmill:dev +FROM ${BASE_IMAGE} # Rust COPY --from=rust:1.97.0 /usr/local/cargo /usr/local/cargo diff --git a/docker/DockerfileFullEe b/docker/DockerfileFullEe index 5b5347558a..88ed619c19 100644 --- a/docker/DockerfileFullEe +++ b/docker/DockerfileFullEe @@ -1,3 +1,5 @@ +ARG BASE_IMAGE=ghcr.io/windmill-labs/windmill-ee:dev + FROM alpine:3.14 AS oracledb-client ARG TARGETPLATFORM @@ -17,7 +19,7 @@ RUN if [ "$TARGETPLATFORM" = "linux/arm64" ]; then \ mv /opt/oracle/23/amd64/lib /opt/oracle/23/lib; \ fi -FROM ghcr.io/windmill-labs/windmill-ee:dev +FROM ${BASE_IMAGE} # Rust COPY --from=rust:1.97.0 /usr/local/cargo /usr/local/cargo diff --git a/docker/DockerfileSlim b/docker/DockerfileSlim index 64fd1b735e..6418a2cfda 100644 --- a/docker/DockerfileSlim +++ b/docker/DockerfileSlim @@ -1,4 +1,7 @@ ARG DEBIAN_IMAGE=debian:trixie-slim +ARG BASE_IMAGE=ghcr.io/windmill-labs/windmill:dev + +FROM ${BASE_IMAGE} AS windmill_image FROM debian:trixie-slim AS nsjail @@ -103,7 +106,7 @@ RUN arch="$(dpkg --print-architecture)"; \ WORKDIR ${APP} -COPY --from=ghcr.io/windmill-labs/windmill:dev --chmod=755 ${APP}/windmill ${APP}/windmill +COPY --from=windmill_image --chmod=755 ${APP}/windmill ${APP}/windmill RUN ln -s ${APP}/windmill /usr/local/bin/windmill diff --git a/docker/DockerfileSlimEe b/docker/DockerfileSlimEe index de0d61a780..24ef941b87 100644 --- a/docker/DockerfileSlimEe +++ b/docker/DockerfileSlimEe @@ -1,4 +1,7 @@ ARG DEBIAN_IMAGE=debian:trixie-slim +ARG BASE_IMAGE=ghcr.io/windmill-labs/windmill-ee:dev + +FROM ${BASE_IMAGE} AS windmill_image FROM debian:trixie-slim AS nsjail @@ -103,7 +106,7 @@ RUN arch="$(dpkg --print-architecture)"; \ WORKDIR ${APP} -COPY --from=ghcr.io/windmill-labs/windmill-ee:dev --chmod=755 ${APP}/windmill ${APP}/windmill +COPY --from=windmill_image --chmod=755 ${APP}/windmill ${APP}/windmill RUN ln -s ${APP}/windmill /usr/local/bin/windmill diff --git a/docs/auth-surface.md b/docs/auth-surface.md index 7a8cbd76ea..a383b9267d 100644 --- a/docs/auth-surface.md +++ b/docs/auth-surface.md @@ -46,15 +46,59 @@ Symbols, not line numbers, are cited: they drift less. (`windmill-api-auth/src/lib.rs`) errors on `authed.job_id.is_some()`. A script that needs `users/create`, `tokens/impersonate`, `set_login_type`, … must use a dedicated superadmin user token stored as a secret, never `$WM_TOKEN`. Token scopes cannot narrow superadmin routes. +- **A remote deploy token is a credential for another instance**, held per account and workspace + (`remote_deploy_token`, encrypted under the workspace key, re-keyed by `set_encryption_key`). + Its `email` references `password(email)` with `ON DELETE/UPDATE CASCADE`, so whatever deletes or + renames an account takes the token along and a recycled address cannot inherit it; removal from + a workspace deletes it explicitly (`delete_workspace_user_internal`, both `leave_workspace`). The + row records the target it was granted for, and + `remote_deploy::proxy` only sends it to a target still matching the workspace setting, so + re-pointing the setting cannot redirect anyone's token to a URL of the admin's choosing. + `require_own_credentials` refuses job, scoped and read-only tokens (on `set_target` too): the + stored token carries none of their restrictions. The proxy turns the local session into a remote + bearer credential, so every ambient-cookie vector becomes one on the remote: its URL carries the + row's random `proxy_key` (a link riding the `SameSite=Lax` cookie cannot know it; a header would + do, but the frontend's only per-call hook is the global `OpenAPI.HEADERS`, whose mere presence + switches every download to in-memory blobs). The key is served `no-store`, withheld from the + credentials `require_own_credentials` refuses, and masked in this instance's own request logs + (`RedactedUri`, used by the request span and the log context); a reverse proxy in front still + writes the full path to its access log. `connect` names the target the token was obtained for + and is refused, before the token is sent anywhere, if the workspace now points elsewhere; the + token only ever goes to that target. It takes no lock against what clears these rows (a removal + from the workspace, a target change, a key rotation): their writers take the account, membership, + key and settings rows in every order, so any lock held there could close a deadlock. A row can + therefore land after one of them ran, however long its remote call took, and every read + (`load_connection`) voids it instead, by identity rather than by comparing times: the connect + records, before its remote call, the target setting's version (`remote_deploy_target_changed_at`) + and the `created_at` of the owner's membership, and the row counts only while both are still + the same (or the owner is a superadmin) and it decrypts. So a target set A → B → A, or a + removal and re-add, voids it whatever the clocks say. A superadmin with no membership is bound + through the credential making the request instead: the insert requires its `token` row to + still exist, and `delete_user`, offboarding and SCIM removal delete an account's tokens, so an + address deleted that way and re-created during the call does not inherit the connection (a JWT, + having no row, cannot connect a workspace its owner is not a member of). `leave_instance` + deletes only the `password` row, leaving the tokens and the memberships, which is not covered. + Neither is a member's connect across an account re-creation: an address is not expected to pass + to another person. A disconnect empties and stamps the row + (inserting one if needed) rather than deleting it, `connected_at` is when a connect started, + and the connect's upsert leaves a row stamped after that alone, so an older connect cannot undo + a disconnect or a newer connect. Connecting by redirect: the remote's + `/user/remote_deploy_authorize` page + mints a token bound to the one remote workspace (`remote-deploy:`, a label + reserved in `is_user_token` so its expiry emails nobody) only on an + explicit Authorize, only for a callback whose path is `/remote_deploy/callback`, and refuses to + render inside a frame; the token travels in the fragment, and the callback checks a single-use + `state` the drawer stored, so no other page can plant a token as the user's. The proxy also refuses a path the URL parser would rewrite, + serves every response under `CSP: sandbox` + `nosniff`, forwards only the method, query, body, + content-type and accept — never this instance's cookie or token — and turns the target's 401 + into a 502, because the browser logs the user out of *this* instance on an unhandled 401. - **`login_type`** (`password` table) is a free-form `VARCHAR(50)`. Password login and password reset require `login_type = 'password'`; `set_password` also accepts `pending_oauth` and turns the account into a `password` one in the same statement (an account created ahead of its owner gets its first credential that way, or through the OAuth claim below). - **Login links** (`login_link` table, `POST /users/login_links` superadmin-only, - `GET /auth/login_link/{token}` unauthenticated): single-use, ≤2 h, a session cookie and a - 302 to a same-origin `rd`. A link minted with `confirm` is the `/user/login_link` page - instead, which spends it only on a click (`POST` to the same path, answering `{location}`), so - a mail scanner opening it does not. `require_login_type` on the mint refuses (409) an account whose + `GET /auth/login_link/{token}` unauthenticated): single-use, ≤15 min, a session cookie and a + 302 to a same-origin `rd`. `require_login_type` on the mint refuses (409) an account whose `login_type` has moved on — the way a caller re-entering an account it created stops being able to once the owner has a password or a provider. - **Pre-approved trial offer** (`cloud_trial_offer`, cloud-only routes under diff --git a/docs/feature-telemetry.md b/docs/feature-telemetry.md index ae62d46b94..9dacfc1cbd 100644 --- a/docs/feature-telemetry.md +++ b/docs/feature-telemetry.md @@ -4,10 +4,10 @@ anonymous usage-stats payload. It answers "does anyone use this, and which variant do they pick" without any identifying data leaving the instance. -It currently carries 51 registered actions across nineteen features (`ai_session`, `ai_chat`, -`ai_fix`, `ai_agent`, `ai_agent_eval`, `app_sandbox`, `datatable`, `flow_editor`, `flow_run`, -`flow_step`, `home`, `run_form`, `debugger`, `trigger`, `command_script`, `hub_script`, -`usage_meter`, `sso_groups_claim`, `cloud_trial_offer`). Nearly all of the +It currently carries 58 registered actions across twenty features (`ai_session`, `ai_chat`, +`ai_fix`, `ai_agent`, `ai_agent_eval`, `app_sandbox`, `datatable`, `db_manager`, `flow_editor`, +`flow_run`, `flow_step`, `home`, `run_form`, `debugger`, `trigger`, `command_script`, +`hub_script`, `usage_meter`, `sso_groups_claim`, `cloud_trial_offer`). Nearly all of the product is uninstrumented, so new user-facing work is the opportunity to change that. ## When to instrument diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 82e2f46d5c..611cd0c91f 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -1,12 +1,12 @@ { "name": "@windmill-labs/components", - "version": "1.815.0", + "version": "1.817.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@windmill-labs/components", - "version": "1.815.0", + "version": "1.817.0", "hasInstallScript": true, "license": "AGPL-3.0", "dependencies": { @@ -19,8 +19,10 @@ "@codingame/monaco-vscode-standalone-json-language-features": "=25.0.0", "@codingame/monaco-vscode-standalone-languages": "=25.0.0", "@codingame/monaco-vscode-standalone-typescript-language-features": "=25.0.0", + "@floating-ui/core": "^1.3.1", "@json2csv/plainjs": "^7.0.6", "@leeoniya/ufuzzy": "^1.0.8", + "@melt-ui/svelte": "^0.86.2", "@popperjs/core": "^2.11.8", "@redocly/json-to-json-schema": "^0.0.1", "@scalar/openapi-parser": "^0.15.0", @@ -29,6 +31,7 @@ "@windmill-labs/svelte-dnd-action": "^0.9.44", "@xterm/addon-fit": "^0.10.0", "@xyflow/svelte": "^1.0.0", + "@zerodevx/svelte-toast": "^0.9.6", "acorn": "^8.15.0", "ag-charts-community": "^9.0.1", "ag-charts-enterprise": "^9.0.1", @@ -39,6 +42,7 @@ "chartjs-adapter-date-fns": "^3.0.0", "chartjs-plugin-zoom": "^2.0.0", "clone": "^2.1.2", + "d3-dag": "^0.11.5", "d3-zoom": "^3.0.0", "date-fns": "^2.30.0", "diff": "^8.0.3", @@ -67,15 +71,23 @@ "panzoom": "^9.4.3", "pdfjs-dist": "4.8.69", "quicktype-core": "^23.2.6", - "quill": "^1.3.7", + "quill": "^2.0.3", "rehype-github-alerts": "^3.0.0", "rehype-raw": "^7.0.0", "rehype-sanitize": "^6.0.0", "rfc4648": "^1.5.3", "runed": "^0.36.0", + "style-to-object": "^0.4.1", + "svelte": "^5.55.7", + "svelte-awesome-color-picker": "^3.0.4", "svelte-carousel": "^1.0.25", "svelte-exmarkdown": "^5.0.0", + "svelte-floating-ui": "^1.5.8", + "svelte-highlight": "^7.6.0", "svelte-infinite-loading": "^1.4.0", + "svelte-popperjs": "^1.3.2", + "svelte-range-slider-pips": "^2.3.1", + "svelte-splitpanes": "^8.0.9", "tailwind-merge": "^1.13.2", "unist-util-visit": "^5.0.0", "vscode": "npm:@codingame/monaco-vscode-extension-api@=25.0.0", @@ -107,10 +119,8 @@ "zod": "^4.1.10" }, "devDependencies": { - "@floating-ui/core": "^1.3.1", "@hey-api/openapi-ts": "^0.43.0", "@melt-ui/pp": "^0.3.2", - "@melt-ui/svelte": "^0.86.2", "@playwright/test": "^1.57.0", "@sveltejs/adapter-static": "^3.0.6", "@sveltejs/kit": "^2.69.1", @@ -124,10 +134,8 @@ "@types/vscode": "^1.83.5", "@typescript-eslint/eslint-plugin": "^5.59.8", "@typescript-eslint/parser": "^5.60.0", - "@zerodevx/svelte-toast": "^0.9.6", "autoprefixer": "^10.4.13", "cssnano": "^6.0.1", - "d3-dag": "^0.11.5", "dts-bundle-generator": "^9.5.1", "eslint": "^8.47.0", "eslint-config-prettier": "^8.6.0", @@ -141,18 +149,10 @@ "postcss-load-config": "^4.0.1", "prettier": "^3.1.0", "prettier-plugin-svelte": "^3.3.3", - "style-to-object": "^0.4.1", "stylelint-config-recommended": "^13.0.0", - "svelte": "^5.55.7", - "svelte-awesome-color-picker": "^3.0.4", "svelte-check": "^4.4.3", "svelte-fast-check": "^0.4.5", - "svelte-floating-ui": "^1.5.8", - "svelte-highlight": "^7.6.0", - "svelte-popperjs": "^1.3.2", "svelte-preprocess": "^6.0.0", - "svelte-range-slider-pips": "^2.3.1", - "svelte-splitpanes": "^8.0.9", "tailwindcss": "^3.4.1", "tar": "^7.5.18", "tslib": "^2.6.1", @@ -1219,7 +1219,6 @@ "version": "1.7.3", "resolved": "https://registry.npmjs.org/@floating-ui/core/-/core-1.7.3.tgz", "integrity": "sha512-sGnvb5dmrJaKEZ+LDIpguvdX3bDlEllmv4/ClQ9awcmCZrlx5jQyyMWFM5kBI+EyNOCDDiKk8il0zeuX3Zlg/w==", - "dev": true, "license": "MIT", "dependencies": { "@floating-ui/utils": "^0.2.10" @@ -1229,7 +1228,6 @@ "version": "1.7.4", "resolved": "https://registry.npmjs.org/@floating-ui/dom/-/dom-1.7.4.tgz", "integrity": "sha512-OOchDgh4F2CchOX94cRVqhvy7b3AFb+/rQXyswmzmGakRfkMgoWVjfnLWkRirfLEfuD4ysVW16eXzwt3jHIzKA==", - "dev": true, "license": "MIT", "dependencies": { "@floating-ui/core": "^1.7.3", @@ -1240,7 +1238,6 @@ "version": "0.2.10", "resolved": "https://registry.npmjs.org/@floating-ui/utils/-/utils-0.2.10.tgz", "integrity": "sha512-aGTxbpbg8/b5JfU1HXSrbH3wXZuLPJcNEcZQFMxLs3oSzgtVu6nFPkbbGGUvBcUjKV2YyB9Wxxabo+HEH9tcRQ==", - "dev": true, "license": "MIT" }, "node_modules/@glideapps/ts-necessities": { @@ -1344,7 +1341,6 @@ "version": "3.10.0", "resolved": "https://registry.npmjs.org/@internationalized/date/-/date-3.10.0.tgz", "integrity": "sha512-oxDR/NTEJ1k+UFVQElaNIk65E/Z83HK1z1WI3lQyhTtnNg4R5oVXaPzK3jcpKG8UHKDVuDQHzn+wsxSz8RP3aw==", - "dev": true, "license": "Apache-2.0", "dependencies": { "@swc/helpers": "^0.5.0" @@ -1576,7 +1572,6 @@ "version": "0.86.6", "resolved": "https://registry.npmjs.org/@melt-ui/svelte/-/svelte-0.86.6.tgz", "integrity": "sha512-Jer+M7DgIwT5IHfTayb4Iw/fkkxWNmC/mqn/nMh9JrbPbkxmyabfLQnhJ+JDn5HK77f84j34lubO3iqFtYAfMg==", - "dev": true, "license": "MIT", "dependencies": { "@floating-ui/core": "^1.3.1", @@ -2160,7 +2155,6 @@ "version": "0.5.17", "resolved": "https://registry.npmjs.org/@swc/helpers/-/helpers-0.5.17.tgz", "integrity": "sha512-5IKx/Y13RsYd+sauPb2x+U/xZikHjolzfuDgTAl/Tdf3Q8rslRvC19NKDLgAJQ6wsqADk10ntlv08nPFw/gO/A==", - "dev": true, "license": "Apache-2.0", "dependencies": { "tslib": "^2.8.0" @@ -3146,7 +3140,6 @@ "version": "0.9.6", "resolved": "https://registry.npmjs.org/@zerodevx/svelte-toast/-/svelte-toast-0.9.6.tgz", "integrity": "sha512-nHlTrCjverlPK9yukK6fqbG3e/R+f10ldrc4nJHOe2qNDScuPTuYVSFEk2dDDtzWAwTN5pmdEXgA3M2RbT8jiw==", - "dev": true, "license": "ISC", "peerDependencies": { "svelte": "^3.57.0 || ^4.0.0 || ^5.0.0" @@ -3746,28 +3739,11 @@ "node": ">=8.10.0" } }, - "node_modules/call-bind": { - "version": "1.0.8", - "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.8.tgz", - "integrity": "sha512-oKlSFMcMwpUg2ednkhQ454wfWiU/ul3CkJe/PEHcTKuiX6RpbehUiFMXu13HalGZxfUwCQzZG747YXBn1im9ww==", - "license": "MIT", - "dependencies": { - "call-bind-apply-helpers": "^1.0.0", - "es-define-property": "^1.0.0", - "get-intrinsic": "^1.2.4", - "set-function-length": "^1.2.2" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, "node_modules/call-bind-apply-helpers": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/call-bind-apply-helpers/-/call-bind-apply-helpers-1.0.2.tgz", "integrity": "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==", + "dev": true, "license": "MIT", "dependencies": { "es-errors": "^1.3.0", @@ -3781,6 +3757,7 @@ "version": "1.0.4", "resolved": "https://registry.npmjs.org/call-bound/-/call-bound-1.0.4.tgz", "integrity": "sha512-+ys997U96po4Kx/ABpBCqhA9EuxJaQWDQg7295H4hBphv3IZg0boBKuwYpt4YXp6MZ5AmZQnU/tyMTlRpaSejg==", + "dev": true, "license": "MIT", "dependencies": { "call-bind-apply-helpers": "^1.0.2", @@ -4150,7 +4127,6 @@ "version": "2.9.3", "resolved": "https://registry.npmjs.org/colord/-/colord-2.9.3.tgz", "integrity": "sha512-jeC1axXpnb0/2nn/Y1LPuLdgXBLH7aDcHu4KEKfqw3CUhX7ZpfBSlPKyqXE6btIgEzfWtrX3/tyBCaCvXvMkOw==", - "dev": true, "license": "MIT" }, "node_modules/combined-stream": { @@ -4676,7 +4652,6 @@ "version": "0.11.5", "resolved": "https://registry.npmjs.org/d3-dag/-/d3-dag-0.11.5.tgz", "integrity": "sha512-sNHvYqjzDlvV2fyEkoOCSuLs2GeWliIg7pJcAiKXgtUSxl0kIX0C2q1J8JzzA9CQWptKxYtzxFCXiKptTW8qsQ==", - "dev": true, "license": "MIT", "dependencies": { "d3-array": "^3.1.6", @@ -5222,26 +5197,6 @@ "dev": true, "license": "MIT" }, - "node_modules/deep-equal": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/deep-equal/-/deep-equal-1.1.2.tgz", - "integrity": "sha512-5tdhKF6DbU7iIzrIOa1AOUt39ZRm13cmL1cGEh//aqR8x9+tNfbywRf0n5FD/18OKMdo7DNEtrX2t22ZAkI+eg==", - "license": "MIT", - "dependencies": { - "is-arguments": "^1.1.1", - "is-date-object": "^1.0.5", - "is-regex": "^1.1.4", - "object-is": "^1.1.5", - "object-keys": "^1.1.1", - "regexp.prototype.flags": "^1.5.1" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, "node_modules/deep-extend": { "version": "0.6.0", "resolved": "https://registry.npmjs.org/deep-extend/-/deep-extend-0.6.0.tgz", @@ -5284,40 +5239,6 @@ "node": ">=6" } }, - "node_modules/define-data-property": { - "version": "1.1.4", - "resolved": "https://registry.npmjs.org/define-data-property/-/define-data-property-1.1.4.tgz", - "integrity": "sha512-rBMvIzlpA8v6E+SJZoo++HAYqsLrkg7MSfIinMPFhmkorw7X+dOXVJQs+QT69zGkzMyfDnIMN2Wid1+NbL3T+A==", - "license": "MIT", - "dependencies": { - "es-define-property": "^1.0.0", - "es-errors": "^1.3.0", - "gopd": "^1.0.1" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/define-properties": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/define-properties/-/define-properties-1.2.1.tgz", - "integrity": "sha512-8QmQKqEASLd5nx0U1B1okLElbUuuttJ/AnYmRXbbbGDWh6uS208EjD4Xqq/I9wK7u0v6O08XhTWnt5XtEbR6Dg==", - "license": "MIT", - "dependencies": { - "define-data-property": "^1.0.1", - "has-property-descriptors": "^1.0.0", - "object-keys": "^1.1.1" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, "node_modules/defu": { "version": "6.1.7", "resolved": "https://registry.npmjs.org/defu/-/defu-6.1.7.tgz", @@ -5570,6 +5491,7 @@ "version": "1.0.1", "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz", "integrity": "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==", + "dev": true, "license": "MIT", "dependencies": { "call-bind-apply-helpers": "^1.0.1", @@ -5684,6 +5606,7 @@ "version": "1.0.1", "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz", "integrity": "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==", + "dev": true, "license": "MIT", "engines": { "node": ">= 0.4" @@ -5693,6 +5616,7 @@ "version": "1.3.0", "resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz", "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==", + "dev": true, "license": "MIT", "engines": { "node": ">= 0.4" @@ -5709,6 +5633,7 @@ "version": "1.1.1", "resolved": "https://registry.npmjs.org/es-object-atoms/-/es-object-atoms-1.1.1.tgz", "integrity": "sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA==", + "dev": true, "license": "MIT", "dependencies": { "es-errors": "^1.3.0" @@ -6052,7 +5977,6 @@ "version": "0.0.3", "resolved": "https://registry.npmjs.org/esm-env-robust/-/esm-env-robust-0.0.3.tgz", "integrity": "sha512-90Gnuw2DALOqlL1581VxP3GHPUNHX9U+fQ+8FNcTTFClhY5gEggAAnJ3q1b2Oq23knRsjv8YpNeMRPaMLUymOA==", - "dev": true, "license": "MIT", "dependencies": { "esm-env": "^1.0.0" @@ -6176,9 +6100,9 @@ } }, "node_modules/eventemitter3": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-2.0.3.tgz", - "integrity": "sha512-jLN68Dx5kyFHaePoXWPsCGW5qdyZQtLYHkxkg02/Mz6g0kYpDx4FyP6XfArhQdlOC4b8Mv+EMxPo/8La7Tzghg==", + "version": "5.0.4", + "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-5.0.4.tgz", + "integrity": "sha512-mlsTRyGaPBjPedk6Bvw+aqbsXDtoAyAzm5MO7JgU+yVRyMQ5O8bD4Kcci7BS85f93veegeCPkL8R4GLClnjLFw==", "license": "MIT" }, "node_modules/events": { @@ -6233,9 +6157,9 @@ "license": "MIT" }, "node_modules/fast-diff": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/fast-diff/-/fast-diff-1.1.2.tgz", - "integrity": "sha512-KaJUt+M9t1qaIteSvjc6P3RbMdXsNhK61GRftR6SNxqmhthcd9MGIi4T+o0jD8LUSpSnSKXE20nLtJ3fOHxQig==", + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/fast-diff/-/fast-diff-1.3.0.tgz", + "integrity": "sha512-VxPP4NqbUjj6MaAOafWeUn2cXWLcCtljklUtZf0Ind4XQ+QPtmA0b18zZy0jIQx+ExRVCR/ZQpBmik5lXshNsw==", "license": "Apache-2.0" }, "node_modules/fast-equals": { @@ -6338,7 +6262,6 @@ "version": "0.7.2", "resolved": "https://registry.npmjs.org/fastpriorityqueue/-/fastpriorityqueue-0.7.2.tgz", "integrity": "sha512-5DtIKh6vtOmEGkYdEPNNb+mxeYCnBiKbK3s4gq52l6cX8I5QaTDWWw0Wx/iYo80fVOblSycHu1/iJeqeNxG8Jw==", - "dev": true, "license": "Apache-2.0" }, "node_modules/fastq": { @@ -6420,7 +6343,6 @@ "version": "7.6.5", "resolved": "https://registry.npmjs.org/focus-trap/-/focus-trap-7.6.5.tgz", "integrity": "sha512-7Ke1jyybbbPZyZXFxEftUtxFGLMpE2n6A+z//m4CRDlj0hW+o3iYSmh8nFlYMurOiJVDmJRilUQtJr08KfIxlg==", - "dev": true, "license": "MIT", "dependencies": { "tabbable": "^6.2.0" @@ -6522,15 +6444,7 @@ "version": "1.1.2", "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==", - "license": "MIT", - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/functions-have-names": { - "version": "1.2.3", - "resolved": "https://registry.npmjs.org/functions-have-names/-/functions-have-names-1.2.3.tgz", - "integrity": "sha512-xckBUXyTIqT97tq2x2AMb+g163b5JFysYk0x4qxNFwbfQkmNZoiRHb6sPzI9/QV33WeuvVYBUIiD4NzNIyqaRQ==", + "dev": true, "license": "MIT", "funding": { "url": "https://github.com/sponsors/ljharb" @@ -6569,6 +6483,7 @@ "version": "1.3.0", "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.3.0.tgz", "integrity": "sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==", + "dev": true, "license": "MIT", "dependencies": { "call-bind-apply-helpers": "^1.0.2", @@ -6593,6 +6508,7 @@ "version": "1.0.1", "resolved": "https://registry.npmjs.org/get-proto/-/get-proto-1.0.1.tgz", "integrity": "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==", + "dev": true, "license": "MIT", "dependencies": { "dunder-proto": "^1.0.1", @@ -6776,6 +6692,7 @@ "version": "1.2.0", "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz", "integrity": "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==", + "dev": true, "license": "MIT", "engines": { "node": ">= 0.4" @@ -6868,22 +6785,11 @@ "node": ">=8" } }, - "node_modules/has-property-descriptors": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/has-property-descriptors/-/has-property-descriptors-1.0.2.tgz", - "integrity": "sha512-55JNKuIW+vq4Ke1BjOTjM2YctQIvCT7GFzHwmfZPGo5wnrgkid0YQtnAleFSqumZm4az3n2BS+erby5ipJdgrg==", - "license": "MIT", - "dependencies": { - "es-define-property": "^1.0.0" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, "node_modules/has-symbols": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.1.0.tgz", "integrity": "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==", + "dev": true, "license": "MIT", "engines": { "node": ">= 0.4" @@ -6896,6 +6802,7 @@ "version": "1.0.2", "resolved": "https://registry.npmjs.org/has-tostringtag/-/has-tostringtag-1.0.2.tgz", "integrity": "sha512-NqADB8VjPFLM2V0VvHUewwwsw0ZWBaIdgo+ieHtK3hasLz4qeCRjYcqfB6AQrBggRKppKF8L52/VqdVsO47Dlw==", + "dev": true, "license": "MIT", "dependencies": { "has-symbols": "^1.0.3" @@ -6917,6 +6824,7 @@ "version": "2.0.4", "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.4.tgz", "integrity": "sha512-T2UbfbBEF32wiepXIsMlTW9+dDYC6wMh/t/vYA4tuOMKqWz/n3vr1NFSxQiyP+zk2mXsoMA/i/7qV6LKut1t1A==", + "dev": true, "license": "MIT", "dependencies": { "function-bind": "^1.1.2" @@ -7303,7 +7211,6 @@ "version": "0.1.1", "resolved": "https://registry.npmjs.org/inline-style-parser/-/inline-style-parser-0.1.1.tgz", "integrity": "sha512-7NXolsK4CAS5+xvdj5OMMbI962hU/wvwoxk+LWR9Ek9bVtyuuYScDN6eS0rUm6TxApFpw7CX1o4uJzcd4AyD3Q==", - "dev": true, "license": "MIT" }, "node_modules/internmap": { @@ -7315,22 +7222,6 @@ "node": ">=12" } }, - "node_modules/is-arguments": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/is-arguments/-/is-arguments-1.2.0.tgz", - "integrity": "sha512-7bVbi0huj/wrIAOzb8U1aszg9kdi3KN/CyU19CTI7tAoZYEZoL9yCDXpbXN+uPsuWnP02cyug1gleqq+TU+YCA==", - "license": "MIT", - "dependencies": { - "call-bound": "^1.0.2", - "has-tostringtag": "^1.0.2" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, "node_modules/is-arrayish": { "version": "0.2.1", "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.2.1.tgz", @@ -7368,22 +7259,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/is-date-object": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/is-date-object/-/is-date-object-1.1.0.tgz", - "integrity": "sha512-PwwhEakHVKTdRNVOw+/Gyh0+MzlCl4R6qKvkhuvLtPMggI1WAHt9sOwZxQLSGpUaDnrdyDsomoRgNnCfKNSXXg==", - "license": "MIT", - "dependencies": { - "call-bound": "^1.0.2", - "has-tostringtag": "^1.0.2" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, "node_modules/is-extglob": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz", @@ -7474,24 +7349,6 @@ "@types/estree": "^1.0.6" } }, - "node_modules/is-regex": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/is-regex/-/is-regex-1.2.1.tgz", - "integrity": "sha512-MjYsKHO5O7mCsmRGxWcLWheFqN9DJ/2TmngvjKXihe6efViPqc274+Fx/4fYj/r03+ESvBdTXK0V6tA3rgez1g==", - "license": "MIT", - "dependencies": { - "call-bound": "^1.0.2", - "gopd": "^1.2.0", - "has-tostringtag": "^1.0.2", - "hasown": "^2.0.2" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, "node_modules/is-url": { "version": "1.2.4", "resolved": "https://registry.npmjs.org/is-url/-/is-url-1.2.4.tgz", @@ -7541,7 +7398,6 @@ "version": "0.4.24", "resolved": "https://registry.npmjs.org/javascript-lp-solver/-/javascript-lp-solver-0.4.24.tgz", "integrity": "sha512-5edoDKnMrt/u3M6GnZKDDIPxOyFOg+WrwDv8mjNiMC2DePhy2H9/FFQgf4ggywaXT1utvkxusJcjQUER72cZmA==", - "dev": true, "license": "Unlicense" }, "node_modules/jiti": { @@ -8686,6 +8542,7 @@ "version": "1.1.0", "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz", "integrity": "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==", + "dev": true, "license": "MIT", "engines": { "node": ">= 0.4" @@ -9896,7 +9753,6 @@ "version": "5.1.16", "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-5.1.16.tgz", "integrity": "sha512-kVrnsrJqMR8+oLJnGEmSWw9BivK5mt7H3FZatVRjrc5wGqFYuBxX1yG7+A7Gi5AefkX6t/oCkizcQgpu0cY1dQ==", - "dev": true, "funding": [ { "type": "github", @@ -10139,31 +9995,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/object-is": { - "version": "1.1.6", - "resolved": "https://registry.npmjs.org/object-is/-/object-is-1.1.6.tgz", - "integrity": "sha512-F8cZ+KfGlSGi09lJT7/Nd6KJZ9ygtvYC0/UYYLI9nmQKLMnydpB9yvbv9K1uSkEu7FU9vYPmVwLg328tX+ot3Q==", - "license": "MIT", - "dependencies": { - "call-bind": "^1.0.7", - "define-properties": "^1.2.1" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/object-keys": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/object-keys/-/object-keys-1.1.1.tgz", - "integrity": "sha512-NuAESUOUMrlIXOfHKzD6bpPu3tYt3xvjNdRIQ+FeT0lNb4K8WR70CaDxhuNguS2XG+GjkyMwOzsN5ZktImfhLA==", - "license": "MIT", - "engines": { - "node": ">= 0.4" - } - }, "node_modules/obug": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/obug/-/obug-2.1.1.tgz", @@ -10356,9 +10187,9 @@ } }, "node_modules/parchment": { - "version": "1.1.4", - "resolved": "https://registry.npmjs.org/parchment/-/parchment-1.1.4.tgz", - "integrity": "sha512-J5FBQt/pM2inLzg4hEWmzQx/8h8D0CiDxaG3vyp9rKrQRSDgBlhjdP5jQGgosEajXPSQouXGHOmVdgo7QmJuOg==", + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/parchment/-/parchment-3.0.0.tgz", + "integrity": "sha512-HUrJFQ/StvgmXRcQ1ftY6VEZUq3jA2t9ncFN4F84J/vN0/FPpQF+8FKXb3l6fLces6q0uOHj6NJn+2xvZnxO6A==", "license": "BSD-3-Clause" }, "node_modules/parent-module": { @@ -11563,7 +11394,6 @@ "version": "1.6.1", "resolved": "https://registry.npmjs.org/quadprog/-/quadprog-1.6.1.tgz", "integrity": "sha512-fN5Jkcjlln/b3pJkseDKREf89JkKIyu6cKIVXisgL6ocKPQ0yTp9n6NZUAq3otEPPw78WZMG9K0o9WsfKyMWJw==", - "dev": true, "license": "MIT", "engines": { "node": ">=8.x" @@ -11677,31 +11507,32 @@ } }, "node_modules/quill": { - "version": "1.3.7", - "resolved": "https://registry.npmjs.org/quill/-/quill-1.3.7.tgz", - "integrity": "sha512-hG/DVzh/TiknWtE6QmWAF/pxoZKYxfe3J/d/+ShUWkDvvkZQVTPeVmUJVu1uE6DDooC4fWTiCLh84ul89oNz5g==", + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/quill/-/quill-2.0.3.tgz", + "integrity": "sha512-xEYQBqfYx/sfb33VJiKnSJp8ehloavImQ2A6564GAbqG55PGw1dAWUn1MUbQB62t0azawUS2CZZhWCjO8gRvTw==", "license": "BSD-3-Clause", "dependencies": { - "clone": "^2.1.1", - "deep-equal": "^1.0.1", - "eventemitter3": "^2.0.3", - "extend": "^3.0.2", - "parchment": "^1.1.4", - "quill-delta": "^3.6.2" + "eventemitter3": "^5.0.1", + "lodash-es": "^4.17.21", + "parchment": "^3.0.0", + "quill-delta": "^5.1.0" + }, + "engines": { + "npm": ">=8.2.3" } }, "node_modules/quill-delta": { - "version": "3.6.3", - "resolved": "https://registry.npmjs.org/quill-delta/-/quill-delta-3.6.3.tgz", - "integrity": "sha512-wdIGBlcX13tCHOXGMVnnTVFtGRLoP0imqxM696fIPwIf5ODIYUHIvHbZcyvGlZFiFhK5XzDC2lpjbxRhnM05Tg==", + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/quill-delta/-/quill-delta-5.1.0.tgz", + "integrity": "sha512-X74oCeRI4/p0ucjb5Ma8adTXd9Scumz367kkMK5V/IatcX6A0vlgLgKbzXWy5nZmCGeNJm2oQX0d2Eqj+ZIlCA==", "license": "MIT", "dependencies": { - "deep-equal": "^1.0.1", - "extend": "^3.0.2", - "fast-diff": "1.1.2" + "fast-diff": "^1.3.0", + "lodash.clonedeep": "^4.5.0", + "lodash.isequal": "^4.5.0" }, "engines": { - "node": ">=0.10" + "node": ">= 12.0.0" } }, "node_modules/rbush": { @@ -11874,26 +11705,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/regexp.prototype.flags": { - "version": "1.5.4", - "resolved": "https://registry.npmjs.org/regexp.prototype.flags/-/regexp.prototype.flags-1.5.4.tgz", - "integrity": "sha512-dYqgNSZbDwkaJ2ceRd9ojCGjBq+mOm9LmtXnAnEGyHhN/5R7iDW2TRw3h+o/jCFxus3P2LfWIIiwowAjANm7IA==", - "license": "MIT", - "dependencies": { - "call-bind": "^1.0.8", - "define-properties": "^1.2.1", - "es-errors": "^1.3.0", - "get-proto": "^1.0.1", - "gopd": "^1.2.0", - "set-function-name": "^2.0.2" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, "node_modules/rehype-github-alerts": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/rehype-github-alerts/-/rehype-github-alerts-3.0.0.tgz", @@ -12290,38 +12101,6 @@ "devOptional": true, "license": "MIT" }, - "node_modules/set-function-length": { - "version": "1.2.2", - "resolved": "https://registry.npmjs.org/set-function-length/-/set-function-length-1.2.2.tgz", - "integrity": "sha512-pgRc4hJ4/sNjWCSS9AmnS40x3bNMDTknHgL5UaMBTMyJnU90EgWh1Rz+MC9eFu4BuN/UwZjKQuY/1v3rM7HMfg==", - "license": "MIT", - "dependencies": { - "define-data-property": "^1.1.4", - "es-errors": "^1.3.0", - "function-bind": "^1.1.2", - "get-intrinsic": "^1.2.4", - "gopd": "^1.0.1", - "has-property-descriptors": "^1.0.2" - }, - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/set-function-name": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/set-function-name/-/set-function-name-2.0.2.tgz", - "integrity": "sha512-7PGFlmtwsEADb0WYyvCMa1t+yke6daIG4Wirafur5kcf+MhUnPms1UeR0CKQdTZD81yESwMHbtn+TR+dMviakQ==", - "license": "MIT", - "dependencies": { - "define-data-property": "^1.1.4", - "es-errors": "^1.3.0", - "functions-have-names": "^1.2.3", - "has-property-descriptors": "^1.0.2" - }, - "engines": { - "node": ">= 0.4" - } - }, "node_modules/setimmediate": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/setimmediate/-/setimmediate-1.0.5.tgz", @@ -12767,7 +12546,6 @@ "version": "0.4.4", "resolved": "https://registry.npmjs.org/style-to-object/-/style-to-object-0.4.4.tgz", "integrity": "sha512-HYNoHZa2GorYNyqiCaBgsxvcJIn7OHq6inEga+E6Ke3m5JkoqpQbnFssk4jwe+K7AhGa2fcha4wSOf1Kn01dMg==", - "dev": true, "license": "MIT", "dependencies": { "inline-style-parser": "0.1.1" @@ -13141,7 +12919,6 @@ "version": "3.1.4", "resolved": "https://registry.npmjs.org/svelte-awesome-color-picker/-/svelte-awesome-color-picker-3.1.4.tgz", "integrity": "sha512-tiFakxvSpCwodOSFW6CflnLVSldSlDY77rDipHfW7hvB+4gTP3y6uO+Dm9O9e7P5N7K+CNyNjjdjbZOHYkMyNQ==", - "dev": true, "license": "MIT", "dependencies": { "colord": "^2.9.3", @@ -13155,7 +12932,6 @@ "version": "1.1.2", "resolved": "https://registry.npmjs.org/svelte-awesome-slider/-/svelte-awesome-slider-1.1.2.tgz", "integrity": "sha512-HFIWwq6rtX6aXyc6ns2R0P6pDLYIr6nRjTPyAddAiTAgZqsohAoEfOEwKKqMmZ6Q+CH48zzcFdyo6lFkBJvyQw==", - "dev": true, "license": "MIT", "peerDependencies": { "svelte": "^3.54.0 || ^4.0.0 || ^5.0.0" @@ -13328,7 +13104,6 @@ "version": "1.6.2", "resolved": "https://registry.npmjs.org/svelte-floating-ui/-/svelte-floating-ui-1.6.2.tgz", "integrity": "sha512-EC+DZtBey50P6l3NSzNQWon3cip8a1bzwdpmCdc45kymqEWL4BKhPemAq7SQ9QLebDPaMECW6YodxFbs2d+O/w==", - "dev": true, "license": "MIT", "dependencies": { "@floating-ui/dom": "^1.6.13" @@ -13338,7 +13113,6 @@ "version": "7.8.4", "resolved": "https://registry.npmjs.org/svelte-highlight/-/svelte-highlight-7.8.4.tgz", "integrity": "sha512-aVp+Q0hH9kI7PlSDrklmFTF4Uj7wYj7UGuqkREnkXlqpEffxr2g6esZcMMaTgECdg5rb2mJqM88+nwS10ecoTg==", - "dev": true, "license": "MIT", "dependencies": { "highlight.js": "11.11.1" @@ -13354,7 +13128,6 @@ "version": "1.3.2", "resolved": "https://registry.npmjs.org/svelte-popperjs/-/svelte-popperjs-1.3.2.tgz", "integrity": "sha512-fwrErlkvngL876WXRnL3OLlfk/n9YkZwwLxuKRpZOYCJLt1zrwhoKTXS+/sRgDveD/zd6GQ35hV89EOip+NBGA==", - "dev": true, "license": "MIT", "peerDependencies": { "@popperjs/core": ">=2", @@ -13421,14 +13194,12 @@ "version": "2.3.1", "resolved": "https://registry.npmjs.org/svelte-range-slider-pips/-/svelte-range-slider-pips-2.3.1.tgz", "integrity": "sha512-P29PNqHld+SiaDuHzf98rLvhSYWXb3TVL9p7U5RG9UX8emUgypZgp9zuIIwpmIXysGQC6JG8duMc5FuaPnSVdg==", - "dev": true, "license": "MPL-2.0" }, "node_modules/svelte-splitpanes": { "version": "8.0.9", "resolved": "https://registry.npmjs.org/svelte-splitpanes/-/svelte-splitpanes-8.0.9.tgz", "integrity": "sha512-L3oLXTC99M191FInTXJ/f/2i0welRql1QuVbPaU8iy6nvCR6X9VyjHCsCpLqKGWHwqkWo/AM9CQ1c0nzlb+MkA==", - "dev": true, "license": "MIT", "dependencies": { "esm-env-robust": "0.0.3" @@ -13538,7 +13309,6 @@ "version": "6.2.0", "resolved": "https://registry.npmjs.org/tabbable/-/tabbable-6.2.0.tgz", "integrity": "sha512-Cat63mxsVJlzYvN51JmVXIgNoUokrIaT2zLclCXjRd8boZ0004U4KCs/sToJ75C6sdlByWxpYnb5Boif1VSFew==", - "dev": true, "license": "MIT" }, "node_modules/table": { diff --git a/frontend/package.json b/frontend/package.json index fa8da23774..46db11793f 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "@windmill-labs/components", - "version": "1.815.0", + "version": "1.817.0", "scripts": { "dev": "vite dev", "dev:ui-builder": "mv static/ui_builder static/ui_builder.dev-disabled 2>/dev/null || true ; trap 'mv static/ui_builder.dev-disabled static/ui_builder 2>/dev/null || true' EXIT ; vite dev", @@ -22,10 +22,8 @@ "test:e2e": "playwright test" }, "devDependencies": { - "@floating-ui/core": "^1.3.1", "@hey-api/openapi-ts": "^0.43.0", "@melt-ui/pp": "^0.3.2", - "@melt-ui/svelte": "^0.86.2", "@playwright/test": "^1.57.0", "@sveltejs/adapter-static": "^3.0.6", "@sveltejs/kit": "^2.69.1", @@ -39,10 +37,8 @@ "@types/vscode": "^1.83.5", "@typescript-eslint/eslint-plugin": "^5.59.8", "@typescript-eslint/parser": "^5.60.0", - "@zerodevx/svelte-toast": "^0.9.6", "autoprefixer": "^10.4.13", "cssnano": "^6.0.1", - "d3-dag": "^0.11.5", "dts-bundle-generator": "^9.5.1", "eslint": "^8.47.0", "eslint-config-prettier": "^8.6.0", @@ -56,18 +52,10 @@ "postcss-load-config": "^4.0.1", "prettier": "^3.1.0", "prettier-plugin-svelte": "^3.3.3", - "style-to-object": "^0.4.1", "stylelint-config-recommended": "^13.0.0", - "svelte": "^5.55.7", - "svelte-awesome-color-picker": "^3.0.4", "svelte-check": "^4.4.3", "svelte-fast-check": "^0.4.5", - "svelte-floating-ui": "^1.5.8", - "svelte-highlight": "^7.6.0", - "svelte-popperjs": "^1.3.2", "svelte-preprocess": "^6.0.0", - "svelte-range-slider-pips": "^2.3.1", - "svelte-splitpanes": "^8.0.9", "tailwindcss": "^3.4.1", "tar": "^7.5.18", "tslib": "^2.6.1", @@ -97,8 +85,10 @@ "@codingame/monaco-vscode-standalone-json-language-features": "=25.0.0", "@codingame/monaco-vscode-standalone-languages": "=25.0.0", "@codingame/monaco-vscode-standalone-typescript-language-features": "=25.0.0", + "@floating-ui/core": "^1.3.1", "@json2csv/plainjs": "^7.0.6", "@leeoniya/ufuzzy": "^1.0.8", + "@melt-ui/svelte": "^0.86.2", "@popperjs/core": "^2.11.8", "@redocly/json-to-json-schema": "^0.0.1", "@scalar/openapi-parser": "^0.15.0", @@ -107,6 +97,7 @@ "@windmill-labs/svelte-dnd-action": "^0.9.44", "@xterm/addon-fit": "^0.10.0", "@xyflow/svelte": "^1.0.0", + "@zerodevx/svelte-toast": "^0.9.6", "acorn": "^8.15.0", "ag-charts-community": "^9.0.1", "ag-charts-enterprise": "^9.0.1", @@ -117,6 +108,7 @@ "chartjs-adapter-date-fns": "^3.0.0", "chartjs-plugin-zoom": "^2.0.0", "clone": "^2.1.2", + "d3-dag": "^0.11.5", "d3-zoom": "^3.0.0", "date-fns": "^2.30.0", "diff": "^8.0.3", @@ -145,15 +137,23 @@ "panzoom": "^9.4.3", "pdfjs-dist": "4.8.69", "quicktype-core": "^23.2.6", - "quill": "^1.3.7", + "quill": "^2.0.3", "rehype-github-alerts": "^3.0.0", "rehype-raw": "^7.0.0", "rehype-sanitize": "^6.0.0", "rfc4648": "^1.5.3", "runed": "^0.36.0", + "style-to-object": "^0.4.1", + "svelte": "^5.55.7", + "svelte-awesome-color-picker": "^3.0.4", "svelte-carousel": "^1.0.25", "svelte-exmarkdown": "^5.0.0", + "svelte-floating-ui": "^1.5.8", + "svelte-highlight": "^7.6.0", "svelte-infinite-loading": "^1.4.0", + "svelte-popperjs": "^1.3.2", + "svelte-range-slider-pips": "^2.3.1", + "svelte-splitpanes": "^8.0.9", "tailwind-merge": "^1.13.2", "unist-util-visit": "^5.0.0", "vscode": "npm:@codingame/monaco-vscode-extension-api@=25.0.0", diff --git a/frontend/scripts/ui_builder_artifact.json b/frontend/scripts/ui_builder_artifact.json index 87ce066c87..68c1c558b1 100644 --- a/frontend/scripts/ui_builder_artifact.json +++ b/frontend/scripts/ui_builder_artifact.json @@ -1,5 +1,5 @@ { "baseUrl": "https://pub-06154ed168a24e73a86ab84db6bf15d8.r2.dev", - "version": "1522e43", - "sha256": "0c44db3a52e02e6530860d0e9e9031fcdad7a09a98897b6c97e6f522c1d4754c" + "version": "60b6904", + "sha256": "85b68e8418c97d96f3a419b3e345ef8a1987976943b8439a7df841f9217a9b34" } diff --git a/frontend/src/lib/aiStore.ts b/frontend/src/lib/aiStore.ts index 6ac98d69e8..ff02157e4a 100644 --- a/frontend/src/lib/aiStore.ts +++ b/frontend/src/lib/aiStore.ts @@ -51,6 +51,8 @@ export const copilotInfo = writable<{ aiModels: AIProviderModel[] customPrompts?: Record maxTokensPerModel?: Record + /** Context windows per `provider:model`, overriding the built-in window table. */ + contextWindowPerModel?: Record /** Negotiated rates per `provider:model`, overriding the built-in price table. */ modelPricing?: Record webSearchEnabledProviders?: Partial> @@ -67,6 +69,7 @@ export const copilotInfo = writable<{ aiModels: [], customPrompts: {}, maxTokensPerModel: {}, + contextWindowPerModel: {}, modelPricing: {}, webSearchEnabledProviders: {} }) @@ -144,6 +147,7 @@ export function setCopilotInfo(aiConfig: AIConfig) { aiModels: aiModels, customPrompts: aiConfig.custom_prompts ?? {}, maxTokensPerModel: aiConfig.max_tokens_per_model ?? {}, + contextWindowPerModel: aiConfig.context_window_per_model ?? {}, webSearchEnabledProviders, modelPricing: aiConfig.model_pricing ?? {}, freeTier: aiConfig.free_tier @@ -160,6 +164,7 @@ export function setCopilotInfo(aiConfig: AIConfig) { aiModels: [], customPrompts: {}, maxTokensPerModel: {}, + contextWindowPerModel: {}, webSearchEnabledProviders: {}, modelPricing: {}, // An exhausted free grant lands here — no providers, but the reason AI is off diff --git a/frontend/src/lib/components/AIProviderPicker.svelte b/frontend/src/lib/components/AIProviderPicker.svelte index 19b1346b84..f69bb6e8f4 100644 --- a/frontend/src/lib/components/AIProviderPicker.svelte +++ b/frontend/src/lib/components/AIProviderPicker.svelte @@ -3,11 +3,13 @@ import Select from './select/Select.svelte' import { fetchAvailableModels, AI_PROVIDERS } from './copilot/lib' import type { AIProvider, ProviderConfig } from '$lib/gen' - import { workspaceStore } from '$lib/stores' import ResourcePicker from './ResourcePicker.svelte' import Toggle from './Toggle.svelte' import { saveConfig, removeConfig, isSameAsStoredConfig } from './aiProviderStorage' import AIReasoningEffortPicker from './AIReasoningEffortPicker.svelte' + import { useOperatingWorkspace } from '$lib/components/operatingWorkspace.svelte' + + const operatingWorkspace = useOperatingWorkspace() interface Props { value: ProviderConfig | undefined @@ -26,7 +28,7 @@ workspace = undefined }: Props = $props() - let effectiveWorkspace = $derived(workspace ?? $workspaceStore ?? '') + let effectiveWorkspace = $derived(workspace ?? $operatingWorkspace ?? '') let value = $derived.by(() => { if (!_uncheckedValue || typeof _uncheckedValue !== 'object') return undefined diff --git a/frontend/src/lib/components/AgentTrace.svelte b/frontend/src/lib/components/AgentTrace.svelte index 8c151e192c..91b87fea67 100644 --- a/frontend/src/lib/components/AgentTrace.svelte +++ b/frontend/src/lib/components/AgentTrace.svelte @@ -2,7 +2,6 @@ import { ExternalLink, Globe } from 'lucide-svelte' import { JobService, type Job } from '$lib/gen' import { base } from '$lib/base' - import { workspaceStore } from '$lib/stores' import { msToReadableTimeShort } from '$lib/utils' import ChatCollapsibleCard from './copilot/chat/ChatCollapsibleCard.svelte' import ToolContentDisplay from './copilot/chat/ToolContentDisplay.svelte' @@ -10,6 +9,9 @@ import GfmMarkdown from './GfmMarkdown.svelte' import type { AgentTraceEntry } from './agentTrace' import { SvelteMap, SvelteSet } from 'svelte/reactivity' + import { useOperatingWorkspace } from '$lib/components/operatingWorkspace.svelte' + + const operatingWorkspace = useOperatingWorkspace() interface Props { entries: AgentTraceEntry[] @@ -40,7 +42,7 @@ try { jobs.set( jobId, - await JobService.getJob({ id: jobId, workspace: workspaceId ?? $workspaceStore! }) + await JobService.getJob({ id: jobId, workspace: workspaceId ?? $operatingWorkspace! }) ) } catch { // A tool job can be gone (retention) or unreadable. The row still has its @@ -121,7 +123,7 @@ {:else if entry.jobId} diff --git a/frontend/src/lib/components/ApiConnectForm.svelte b/frontend/src/lib/components/ApiConnectForm.svelte index dedeee90e2..2255817c5e 100644 --- a/frontend/src/lib/components/ApiConnectForm.svelte +++ b/frontend/src/lib/components/ApiConnectForm.svelte @@ -1,7 +1,6 @@ @@ -20,7 +22,7 @@ Setup the wmill cli for this workspace & remote:
diff --git a/frontend/src/lib/components/DBManager.svelte b/frontend/src/lib/components/DBManager.svelte index 7520cc5013..16e64dc5a0 100644 --- a/frontend/src/lib/components/DBManager.svelte +++ b/frontend/src/lib/components/DBManager.svelte @@ -1,11 +1,18 @@ - +
- {#if dbSelector} - {@render dbSelector()} - {/if} - {#if dbSupportsSchemas && !multiSelectMode} - e.stopPropagation()} - onchange={() => toggleSchemaSelection(schemaKey)} - /> - - {/if} - {schemaKey} - - {schemaTables.length} - - - -
- - {#each schemaTables as tableKey} - {@const isDisabled = isTableDisabled(schemaKey, tableKey)} - {@const isChecked = isTableSelected(schemaKey, tableKey) || isDisabled} - {@const isCurrentPreview = - selected.schemaKey === schemaKey && selected.tableKey === tableKey} -
{ - selectTable(schemaKey, tableKey) - toggleTableSelection(schemaKey, tableKey) - }} - onkeydown={(e) => { - if (e.key === 'Enter' || e.key === ' ') { - selectTable(schemaKey, tableKey) - toggleTableSelection(schemaKey, tableKey) - } - }} - > - - e.stopPropagation()} - onchange={() => toggleTableSelection(schemaKey, tableKey)} - /> - - -

{tableKey}

- + {#if dtOpen} + {#if root.error} +

{root.error}

+ {/if} + {#each root.schemas as sc (sc.schemaKey)} + {@const schemaOpen = isExpanded(root.datatable, sc.schemaKey)} + {@const indent = root.datatable !== undefined ? 'pl-7' : 'pl-3'} + {#if dbSupportsSchemas} -
- {/each} - - - {/each} - {:else} - - {#each filteredTableKeys as tableKey} - - - {/each} - {/if} + ]} + btnId={'db-manager-schema-actions-' + onlyAlphaNumAndUnderscore(sc.schemaKey)} + /> + {/if} + + + {/if} + + + {#if schemaOpen || !dbSupportsSchemas} + {@const tableIndent = dbSupportsSchemas + ? root.datatable !== undefined + ? 'pl-11' + : 'pl-7' + : root.datatable !== undefined + ? 'pl-7' + : 'pl-3'} + {#each sc.tables as tableKey (tableKey)} + {@const entry = { + datatable: root.datatable, + schema: sc.schemaKey, + table: tableKey + }} + {@const hasMenu = !multiSelectMode} + {@const isSelected = + root.datatable === currentDatatable && + selected.schemaKey === sc.schemaKey && + selected.tableKey === tableKey} + + {/each} + {#if canCreateTableIn(root.datatable, sc.schemaKey)} + + {/if} + {/if} + + {/each} + {#if dbSupportsSchemas && search.trim() === '' && canCreateSchemaIn(root.datatable)} + + {/if} + {/if} + {/each} - {#if !multiSelectMode} - - {/if} - - {#if tableKey && colDefs?.[tableKey]?.length} + + {#if mainPane} + {@render mainPane()} + {:else if viewMode === 'diagram'} + { + onViewMode?.('data') + selectTable(currentDatatable, schema, table) + }} + /> + {:else if tableKey && colDefs?.[tableKey]?.length} {@const dbTableOps = dbTableOpsFactory({ colDefs: colDefs[tableKey], tableKey, whereClause })} - {#if !!$userStore?.is_admin || !!$superadmin} + {#if !!actingUser?.is_admin || !!$superadmin} {/snippet} diff --git a/frontend/src/lib/components/DBManagerContent.svelte b/frontend/src/lib/components/DBManagerContent.svelte index b0ef2859df..b88b1d1510 100644 --- a/frontend/src/lib/components/DBManagerContent.svelte +++ b/frontend/src/lib/components/DBManagerContent.svelte @@ -1,16 +1,21 @@ + + (open = false) }} +> + { + // The row underneath folds on click, and picking a role is not that. + e.stopPropagation() + open = !open + }} + > + + {role} + + + anchorEl!.getBoundingClientRect())} + onSelectValue={(item) => { + open = false + if (item.value !== role) onSelect(item.value) + }} + /> + diff --git a/frontend/src/lib/components/DdlMigrationGuard.svelte b/frontend/src/lib/components/DdlMigrationGuard.svelte index 3bda6aa216..0dab262fd0 100644 --- a/frontend/src/lib/components/DdlMigrationGuard.svelte +++ b/frontend/src/lib/components/DdlMigrationGuard.svelte @@ -6,8 +6,18 @@ import { joinSqlStatements, splitSqlRuns } from './sqlDdl' import { logDdlGuardChoice } from './workspaceSettings/datatableTelemetry' import { CornerDownLeft } from 'lucide-svelte' + import { withMigrationRole } from './datatableMigrationRole' - let { workspace, datatable }: { workspace: string; datatable: string } = $props() + let { + workspace, + datatable, + role + }: { + workspace: string + datatable: string + /** The role the editor runs as. The migration declares it, or it would run as admin. */ + role?: string + } = $props() type Choice = 'run' | 'migrate' | 'cancel' @@ -73,7 +83,7 @@ function openMigrationModal(sql: string): Promise { return new Promise((resolve) => { resolveMigrationClosed = (created: boolean) => resolve(created) - newMigrationModal?.open({ codeUp: sql }) + newMigrationModal?.open({ codeUp: withMigrationRole(sql, role) }) }) } @@ -145,6 +155,11 @@ migrations rather than run ad-hoc. Create a migration for it instead? {/if}

+ {#if role} +

+ It will run as role {role}. +

+ {/if} 
{promptSql}
- import { userStore } from '$lib/stores' import { SettingsIcon } from 'lucide-svelte' import { Button } from './common' import Drawer from './common/drawer/Drawer.svelte' import DrawerContent from './common/drawer/DrawerContent.svelte' import DefaultScriptsInner from './DefaultScriptsInner.svelte' + import { useOperatingUser } from '$lib/components/operatingWorkspace.svelte' + + const operatingUser = useOperatingUser() + const actingUser = $derived(operatingUser.current) interface Props { placement?: 'left' | 'right' @@ -17,7 +20,7 @@ let drawer: Drawer | undefined = $state() -{#if $userStore?.is_admin || $userStore?.is_super_admin} +{#if actingUser?.is_admin || actingUser?.is_super_admin} diff --git a/frontend/src/lib/components/DefaultScriptsInner.svelte b/frontend/src/lib/components/DefaultScriptsInner.svelte index eaa3d989a0..b062fb4c49 100644 --- a/frontend/src/lib/components/DefaultScriptsInner.svelte +++ b/frontend/src/lib/components/DefaultScriptsInner.svelte @@ -1,10 +1,13 @@ + + +
+
+ It was saved from another tab or session since this one read it, so your changes here are no + longer being saved. +
+
+ + +
+
+ diff --git a/frontend/src/lib/components/DropdownSubmenuItem.svelte b/frontend/src/lib/components/DropdownSubmenuItem.svelte index 60f7ad2d64..a67411a039 100644 --- a/frontend/src/lib/components/DropdownSubmenuItem.svelte +++ b/frontend/src/lib/components/DropdownSubmenuItem.svelte @@ -13,9 +13,10 @@ item: Item builders: ReturnType['builders'] meltItem: MenubarMenuElements['item'] + close?: (afterClose?: () => void) => void } - let { item, builders, meltItem }: Props = $props() + let { item, builders, meltItem, close }: Props = $props() const { elements: { subTrigger, subMenu }, @@ -23,6 +24,8 @@ } = untrack(() => builders).createSubmenu() let subItems = $derived((item.submenuItems ?? []).filter((i) => !i.hide)) + const CustomSubmenu = $derived(item.customSubmenu) + const closeMenu = $derived(close ?? (() => {})) + +{:else} +
+

+ Deploys to {target.workspace_id} on {target.base_url} run with your own account on that instance, + so its permissions, protection rules and audit logs apply. Sign in there to connect; the token + it issues is stored encrypted and only ever sent to that instance. +

+
+ + {#if waitingForRemote} + Waiting for {host}… + {/if} + +
+ {#if showPaste} +
+ + Create a token on {host} + + +
+
+ e.key == 'Enter' && connect()} + /> +
+ +
+
+ {/if} + {#if error} +

{error}

+ {/if} +
+{/if} diff --git a/frontend/src/lib/components/RemoteDeployTargetSetting.svelte b/frontend/src/lib/components/RemoteDeployTargetSetting.svelte new file mode 100644 index 0000000000..91f876ec17 --- /dev/null +++ b/frontend/src/lib/components/RemoteDeployTargetSetting.svelte @@ -0,0 +1,111 @@ + + + + {#if !$enterpriseLicense} + + Deploying to another instance from the web UI is only available with an enterprise license + + {:else} +
+ + +
+ + {#if saved} + + {/if} +
+ {#if saved} +

+ Changing the instance URL or workspace drops every token stored for the current target, so + everyone connects again. +

+ {/if} +
+ {/if} + diff --git a/frontend/src/lib/components/ResourceEditor.svelte b/frontend/src/lib/components/ResourceEditor.svelte index bcb886ee7d..9847454aca 100644 --- a/frontend/src/lib/components/ResourceEditor.svelte +++ b/frontend/src/lib/components/ResourceEditor.svelte @@ -9,7 +9,6 @@ } from '$lib/gen' import { canWrite } from '$lib/utils' import { createEventDispatcher, onDestroy, untrack } from 'svelte' - import { workspaceStore } from '$lib/stores' import { sendUserToast } from '$lib/toast' import { clearJsonSchemaResourceCache } from './schema/jsonSchemaResource.svelte' import ResourceForm from './ResourceForm.svelte' @@ -20,8 +19,13 @@ import { useActingUser } from '$lib/actingUser.svelte' import { UserDraft, draftValuesEqual, type UserDraftHandle } from '$lib/userDraft.svelte' import { UserDraftDbSyncer } from '$lib/userDraftDbSyncer.svelte' + import { useDraftConflictSession } from '$lib/draftConflictSession.svelte' + import DraftConflictAlert from './DraftConflictAlert.svelte' import { setLocalDraftHint } from '$lib/localDraftHints.svelte' import { onUserInput } from '$lib/userDraftEditGate' + import { useOperatingWorkspace } from '$lib/components/operatingWorkspace.svelte' + + const operatingWorkspace = useOperatingWorkspace() interface Props { canSave?: boolean @@ -46,6 +50,9 @@ * so it can hide the banner's Discard button in read-only mode (matches * the trigger editors' `disabled={!can_write}` wiring). */ onCanWriteChange?: (canWrite: boolean) => void + /** The drawer renders the conflict alert: inside this scrollable form it can land above the + * viewport on a long resource, leaving the fixed banner claiming the edits are saved. */ + onDraftConflictChange?: (state: { conflicted: boolean; busy: boolean }) => void } let { @@ -59,7 +66,8 @@ selected: selectedProp = $bindable(), viewJsonSchema = $bindable(), onDraftStateChange, - onCanWriteChange + onCanWriteChange, + onDraftConflictChange }: Props = $props() type ResourceState = { @@ -74,7 +82,7 @@ // Sole ambient read in this file: the acting workspace is an input, and only its // default comes from the navigation store. - let effectiveWorkspace = $derived(workspace ?? $workspaceStore!) + let effectiveWorkspace = $derived(workspace ?? $operatingWorkspace!) // Fallback to `effectiveWorkspace` insulates against reactify-style // parents that re-spread props without `selected` — otherwise it // transiently resets and the form below remounts on every keystroke. @@ -261,6 +269,132 @@ ) const anyDirty = $derived(dirtyWorkspaces.length > 0) + /** Scoped by `selected` so switching workspace-specific versions is a new session. Ending is + * exported on top of that because the drawer, not this component, knows when its own session + * is over: this component outlives it. */ + const conflictSession = useDraftConflictSession(() => selected) + export function endEditingSession(): void { + conflictSession.end() + } + onDestroy(endEditingSession) + const resolvingConflict = $derived(conflictSession.busy) + /** The server refused this tab's autosave because the row moved under it: another tab, or the + * AI chat, which writes these drafts too. Nothing typed here reaches the server until the user + * picks a version, and the unsaved-changes banner says the opposite — that the edits are held + * as a draft — so without this they are told their work is safe while it is being dropped. */ + const draftConflict = $derived( + selected && initialPath + ? UserDraftDbSyncer.getConflict({ + workspace: selected, + itemKind: 'resource', + path: initialPath + }).conflict + : undefined + ) + + async function resolveDraftConflict(keepMine: boolean): Promise { + const ws = selected + const p = initialPath + if (!ws || !p || resolvingConflict) return + const query = { workspace: ws, itemKind: 'resource' as const, path: p } + const token = conflictSession.start() + const stillOurs = () => conflictSession.holds(token) && selected === ws + try { + if (keepMine) { + // Settle the key first: an ordinary autosave still queued would displace the forced + // write below, and being conditional it would be refused — so "Keep mine" would + // finish without keeping anything and leave the alert standing. + await UserDraftDbSyncer.quiesce(query) + // A resolution belongs to the session that started it. One that outlives its editor + // stops here rather than writing on: whatever replaced it — another session on the + // same draft, or its own resolution — owns the key now, and the edit this one was + // keeping is still parked for a later flush either way. + if (!stillOurs()) return + // A parked `null` is this tab's "no draft any more" — a discard, or an edit that + // landed back on the deployed value. Keeping that means removing the row, not + // writing the baseline back as a draft with no dirty banner to discard it through. + const parked = UserDraftDbSyncer.peekPending(query) + const mine = parked?.value === null ? null : $state.snapshot(states[ws]?.draft) + // Forced, so it goes over the row that refused us, and its response reseeds + // `last_sync` so the next ordinary save is conditional again. + if (mine !== undefined) await UserDraftDbSyncer.overwrite({ ...query, value: mine }) + // Say so rather than leave the alert up with no explanation: a write displaced by + // something typed meanwhile can still lose the race. + if (UserDraftDbSyncer.getConflict(query).conflict) { + sendUserToast('Could not keep your version — try again', true) + } + return + } + // Settle the key BEFORE reading, so what comes back is the version the server is left + // holding: a write this tab started can still be in flight — a forced one it walked + // away from included — and a response fetched past it describes a version about to be + // replaced, which would then be seeded along with its already-stale `last_sync`. + // Nothing is given up by waiting; `quiesce` only stops the pipeline. + await UserDraftDbSyncer.quiesce(query) + if (!stillOurs()) return + // Read BEFORE giving anything up: until the server has answered, the refused payload is + // still the only copy of this tab's edit, and the conflict is still true. + const r = await ResourceService.getResource({ workspace: ws, path: p, getDraft: true }) + const deployedState: ResourceState = { + path: r.path, + args: (r.value ?? {}) as Record, + description: r.description ?? '', + labels: r.labels ?? undefined, + wsSpecific: r.ws_specific ?? false + } + // Everything below writes shared editor state, so first make sure it is still this + // resource's: the drawer stays closable while the read is out, and another resource + // opened meanwhile would otherwise get this one's baseline — and with it this one's + // path as its save target. + if (!stillOurs()) return + // Again, because the form stayed editable while the read was out: `quiesce` settles what + // is running when it is called, not the key for the rest of the resolution, so a + // keystroke since can have started a save of its own. Left running, its rejection lands + // after the baseline below and raises the conflict this just resolved. + await UserDraftDbSyncer.quiesce(query) + if (!stillOurs()) return + // Now, and not in `quiesce`: the refused payload belongs to the version being replaced, + // but until this point it was still the only copy of the edit, and a resolution that + // gave up before here has to leave it behind. + UserDraftDbSyncer.dropPending(query) + UserDraftDbSyncer.clearConflict(query) + initialStates[ws] = structuredClone(deployedState) + // Everything else the load path takes from this same response. The item can have been + // deleted, recreated under another type, or had its permissions changed while the + // conflict stood, and the fields below decide create-vs-update, the schema and write + // access — so refreshing only what is displayed would leave those deciding on the + // version the user just replaced. + fetchedResources[ws] = r + existedInitially[ws] = !(r as any).no_deployed + if (ws === effectiveWorkspace) resource_type = r.resource_type + UserDraftDbSyncer.recordRemoteSync(query, (r as any).draft_saved_at) + const loadedDraft = (r as any).draft as ResourceState | undefined + // Loading a draft makes this workspace one that opened with a draft, whatever it opened + // with before — a refused deletion opens gated, and gated the settling absorber would + // fold the version just loaded into the deployed baseline, leaving it silently clean + // with Save disabled. Ungate here rather than leaving it to the effect, so no write + // between the two is absorbed. + openedOnDraft[ws] = !!loadedDraft + if (loadedDraft) { + setGated(ws, false) + } else { + // Accepting "there is no draft" has to shut the gate, the way discarding one does. + // The edit that raised the conflict set `userEdited`, and left open, the form's + // settling writes — schema defaults materializing over the loaded value — read as + // the user's and recreate the draft just accepted away. + userEdited[ws] = false + setGated(ws, true) + } + UserDraft.seed('resource', p, loadedDraft ?? deployedState, { workspace: ws }) + } catch (e) { + // Nothing was given up above, so the conflict stands and the edit is still here to + // resolve again — which is the whole point of reading first. + sendUserToast(`Could not load the other version: ${e}`, true) + } finally { + conflictSession.finish(token) + } + } + // The syncer owns the list-page `*` hint; the editor only CLEARS it when a // workspace is at the deployed baseline (so a draft discarded elsewhere // vanishes on reopen). Never SET here. See VariableEditor for the full note. @@ -336,15 +470,33 @@ labels: r.labels ?? undefined, wsSpecific: r.ws_specific ?? false } - // Open with the saved draft if present, else the deployed. - const s: ResourceState = savedDraftState ?? deployedState - openedOnDraft[ws] = !!savedDraftState + // A refused save leaves this tab's own version parked. `.draft` is the + // version that refused it, so opening on that would quietly drop the edit + // the alert is about and leave "Keep mine" offering to keep the other one. + const conflictQuery = { + workspace: ws, + itemKind: 'resource' as const, + path: initialPath + } + const refused = UserDraftDbSyncer.getConflict(conflictQuery).conflict + ? UserDraftDbSyncer.peekPending(conflictQuery) + : undefined + // A parked `null` is this tab's "no draft any more", which on screen is the + // deployed value — so only a payload with content counts as a local draft. + const refusedDraft = (refused?.value ?? undefined) as ResourceState | undefined + const hasLocalDraft = refused ? refused.value !== null : !!savedDraftState + // Open with this tab's refused version if there is one, else the saved + // draft, else the deployed. + const s: ResourceState = refused + ? (refusedDraft ?? deployedState) + : (savedDraftState ?? deployedState) + openedOnDraft[ws] = hasLocalDraft // Gate BEFORE the handle is acquired: `stopSync` queues on a // not-yet-live entry, and the form can settle before the effect // above gets a chance to run. Only worth doing when no draft exists // yet — where one does, there is no phantom to prevent and // suspending could only drop a write. - if (!savedDraftState) setGated(ws, true) + if (!hasLocalDraft) setGated(ws, true) ensureHandle(ws, s) initialStates[ws] = structuredClone(deployedState) // Draft-only paths (`no_deployed`) have no row — saving must @@ -424,6 +576,13 @@ $effect(() => { onCanWriteChange?.(can_write === true) }) + $effect(() => { + onDraftConflictChange?.({ conflicted: !!draftConflict, busy: resolvingConflict }) + }) + + export function resolveDraftConflictFromBanner(keepMine: boolean): void { + void resolveDraftConflict(keepMine) + } export function localDraftDeployed(): ResourceState | undefined { return selected ? initialStates[selected] : undefined @@ -471,6 +630,12 @@ current.path = npath } + /** The path the resource has in `ws`: after a save, the one it was saved under. Each + * workspace-specific version keeps its own, so the selected one says nothing about `ws`. */ + export function pathIn(ws: string): string | undefined { + return initialStates[ws]?.path + } + /** Whether the write landed. It toasts its own failure, so most callers ignore this; * one that follows the save with bookkeeping of its own has to know not to. */ export async function save(): Promise { @@ -531,6 +696,18 @@
+ + {#if draftConflict && !onDraftConflictChange} + void resolveDraftConflict(false)} + onOverwrite={() => void resolveDraftConflict(true)} + /> + {/if} + {#if otherDirty.length > 0} You are going to edit the value in: {otherDirty.join(', ')} diff --git a/frontend/src/lib/components/ResourceEditorDrawer.svelte b/frontend/src/lib/components/ResourceEditorDrawer.svelte index 052c8c9e92..ce4d1eea5a 100644 --- a/frontend/src/lib/components/ResourceEditorDrawer.svelte +++ b/frontend/src/lib/components/ResourceEditorDrawer.svelte @@ -5,13 +5,14 @@ import { History, Loader2, Save } from 'lucide-svelte' import WsSpecificVersions from './WsSpecificVersions.svelte' - import { workspaceStore } from '$lib/stores' import { isOwner } from '$lib/utils' import { useActingUser } from '$lib/actingUser.svelte' import LocalDraftBanner from './LocalDraftBanner.svelte' + import DraftConflictAlert from './DraftConflictAlert.svelte' import OpenInSessionButton from './sessions/OpenInSessionButton.svelte' import { clearPageDrawerAnchor, + handOffPageDrawer, pageDrawerSessionSource, setPageDrawerAnchor } from './sessions/pageDrawerSession' @@ -20,10 +21,15 @@ import IconedResourceType from './IconedResourceType.svelte' import { addResourceTitle } from './resourceTypeDisplay' import { loadResourceTypeDisplayName } from './displayNameLoaders' + import { useOperatingWorkspace } from '$lib/components/operatingWorkspace.svelte' + + const operatingWorkspace = useOperatingWorkspace() let { workspace = undefined, disableChatOffset = false, + inline = false, + onClose = undefined, onRestored = undefined, onSaved = undefined }: { @@ -31,10 +37,17 @@ * once, at `effectiveWorkspace`, and nowhere else in this file. */ workspace?: string disableChatOffset?: boolean + /** Render in place, filling the parent, with no drawer or close button — for a host + * that gives the editor a whole pane. Saving and restoring then leave it open: the + * host remounts it on what was written. */ + inline?: boolean + /** With `inline`, closes whatever hosts the editor; the header has a close button only when set. */ + onClose?: () => void onRestored?: () => void /** Fires after Save has written, for a caller showing state derived from the - * resource — `onRestored` only covers restoring an old version. */ - onSaved?: () => void + * resource — `onRestored` only covers restoring an old version. `path` is where the + * resource now lives in this drawer's workspace, undefined when the save failed. */ + onSaved?: (path: string | undefined) => void } = $props() let drawer: Drawer | undefined = $state() @@ -45,20 +58,24 @@ let resourceEditor: | { - save: () => void + save: () => Promise + pathIn: (ws: string) => string | undefined localDraftDeployed: () => unknown localDraftCurrent: () => unknown discardLocalDraft: () => void + endEditingSession: () => void + resolveDraftConflictFromBanner: (keepMine: boolean) => void } | undefined = $state(undefined) let hasLocalDraft = $state(false) + let draftConflict = $state({ conflicted: false, busy: false }) let canWriteSelected = $state(true) let path: string | undefined = $state(undefined) let selected: string | undefined = $state(undefined) let viewJsonSchema = $state(false) - let effectiveWorkspace = $derived(workspace ?? $workspaceStore!) + let effectiveWorkspace = $derived(workspace ?? $operatingWorkspace!) // The editor renders whichever workspace-specific variant `selected` points at, so history has // to follow it too — otherwise a restore would write over the variant the user is not looking at. let historyWorkspace = $derived(selected ?? effectiveWorkspace) @@ -85,6 +102,7 @@ * dedicated editor elsewhere: the generic form would render its configuration field by field, * and materialize a default into every one the value leaves out. */ export async function initEdit(p: string, opts?: { json?: boolean }): Promise { + if (handOffPageDrawer(RESOURCES_PATH, p)) return // A `close({ keepAnchor })` on an already-closed drawer emits no close event, so the flag // would still be standing when the next drawer session ends and would swallow that one's // anchor clear. Every session starts having to clear its own. @@ -126,22 +144,46 @@ ) - { - if (keepAnchorOnClose) { - keepAnchorOnClose = false - return - } - clearPageDrawerAnchor(RESOURCES_PATH) - }} -> +{#if inline} + + {#if path !== undefined || resource_type !== undefined} + {@render content()} + {/if} +{:else} + { + // The editor outlives this drawer, so tell it the session is over: a conflict resolution + // still in flight must not land on whatever the next opening shows. + resourceEditor?.endEditingSession?.() + if (keepAnchorOnClose) { + keepAnchorOnClose = false + return + } + clearPageDrawerAnchor(RESOURCES_PATH) + }} + > + {@render content()} + +{/if} + +{#snippet content()} { + // Inline has no drawer to emit a close, so the session ends here instead. + if (inline) { + resourceEditor?.endEditingSession?.() + onClose?.() + } else { + drawer?.closeDrawer() + } + }} > {#snippet titleExtra()} {#if mode == 'new' && resource_type} @@ -162,10 +204,18 @@ bind:selected bind:viewJsonSchema onDraftStateChange={(v) => (hasLocalDraft = v)} + onDraftConflictChange={(v) => (draftConflict = v)} onCanWriteChange={(v) => (canWriteSelected = v)} /> {/await} {#snippet banner()} + {#if draftConflict.conflicted} + resourceEditor?.resolveDraftConflictFromBanner?.(false)} + onOverwrite={() => resourceEditor?.resolveDraftConflictFromBanner?.(true)} + /> + {/if} @@ -212,7 +262,7 @@ {/snippet} - +{/snippet} diff --git a/frontend/src/lib/components/ResourceForm.svelte b/frontend/src/lib/components/ResourceForm.svelte index df878280a7..1d4f091236 100644 --- a/frontend/src/lib/components/ResourceForm.svelte +++ b/frontend/src/lib/components/ResourceForm.svelte @@ -8,7 +8,7 @@ import { Alert, Skeleton } from './common' import Path from './Path.svelte' import LabelsInput from './LabelsInput.svelte' - import { workspaceStore, type UserExt } from '$lib/stores' + import { type UserExt } from '$lib/stores' import SchemaForm from './SchemaForm.svelte' import SimpleEditor from './SimpleEditor.svelte' import FilesetEditor from './FilesetEditor.svelte' @@ -22,6 +22,9 @@ import SyncResourceTypes from './SyncResourceTypes.svelte' import Label from './Label.svelte' import ResourcePathHint from './ResourcePathHint.svelte' + import { useOperatingWorkspace } from '$lib/components/operatingWorkspace.svelte' + + const operatingWorkspace = useOperatingWorkspace() interface Props { path: string @@ -81,7 +84,7 @@ onCredentialStored }: Props = $props() - let ws = $derived(workspace ?? $workspaceStore) + let ws = $derived(workspace ?? $operatingWorkspace) let rawCode: string | undefined = $state(undefined) let textFileContent: string = $state('') diff --git a/frontend/src/lib/components/ResourcePicker.svelte b/frontend/src/lib/components/ResourcePicker.svelte index 06023b1bb4..778c1d2614 100644 --- a/frontend/src/lib/components/ResourcePicker.svelte +++ b/frontend/src/lib/components/ResourcePicker.svelte @@ -1,6 +1,5 @@ - clearPageDrawerAnchor(VARIABLES_PATH)}> +{#if inline} + {@render content()} +{:else} + { + endEditingSession() + clearPageDrawerAnchor(VARIABLES_PATH) + }} + > + {@render content()} + +{/if} + +{#snippet content()} { + // Inline has no drawer to emit a close, so the session ends here instead. + if (inline) { + endEditingSession() + onClose?.() + } else { + drawer?.closeDrawer() + } + }} > {#snippet banner()} + {#if draftConflict} + void resolveDraftConflict(false)} + onOverwrite={() => void resolveDraftConflict(true)} + /> + {/if} {/snippet} - +{/snippet} diff --git a/frontend/src/lib/components/VariableForm.svelte b/frontend/src/lib/components/VariableForm.svelte index 61945eef08..63fdbf7425 100644 --- a/frontend/src/lib/components/VariableForm.svelte +++ b/frontend/src/lib/components/VariableForm.svelte @@ -10,10 +10,13 @@ import ToggleButton from './common/toggleButton-v2/ToggleButton.svelte' import { Loader2, RotateCcw } from 'lucide-svelte' import autosize from '$lib/autosize' - import { workspaceStore, type UserExt } from '$lib/stores' + import { type UserExt } from '$lib/stores' import { isOwner } from '$lib/utils' import { isEncryptedDraftValue } from '$lib/encryptedDraft' import EncryptedDraftField from './EncryptedDraftField.svelte' + import { useOperatingWorkspace } from '$lib/components/operatingWorkspace.svelte' + + const operatingWorkspace = useOperatingWorkspace() interface Variable { value: string @@ -56,7 +59,7 @@ actingUser }: Props = $props() - let ws = $derived(workspace ?? $workspaceStore) + let ws = $derived(workspace ?? $operatingWorkspace) // Loading the deployed secret overwrites the draft row this form shares with the AI // chat, so every path that would trigger it has to be blocked while that row stages a diff --git a/frontend/src/lib/components/WorkerTagPicker.svelte b/frontend/src/lib/components/WorkerTagPicker.svelte index ccc3266ced..dae4967e05 100644 --- a/frontend/src/lib/components/WorkerTagPicker.svelte +++ b/frontend/src/lib/components/WorkerTagPicker.svelte @@ -2,6 +2,7 @@ import { Button } from '$lib/components/common' import { ExternalLink, RotateCw, Loader2 } from 'lucide-svelte' import { workerTags, workspaceStore } from '$lib/stores' + import { useOperatingWorkspace } from '$lib/components/operatingWorkspace.svelte' import AssignableTags from './AssignableTags.svelte' import { WorkerService } from '$lib/gen' import WorkerTagSelect from './WorkerTagSelect.svelte' @@ -11,8 +12,8 @@ popupPlacement?: 'bottom-end' | 'top-end' disabled?: boolean placeholder?: string - // Workspace to read tags from; defaults to $workspaceStore. A fork-scoped - // session passes its effective workspace so the picker matches the deploy target. + // Workspace to read tags from; defaults to the operating workspace (see + // `useOperatingWorkspace`). workspaceId?: string } @@ -26,8 +27,11 @@ // See WorkerTagSelect: the shared `workerTags` cache is navigation-scoped, so a // different target workspace reads/writes a local list to avoid clobbering it. - let effectiveWorkspace = $derived(workspaceId ?? $workspaceStore) - let usesLocal = $derived(workspaceId != undefined && workspaceId !== $workspaceStore) + const operatingWorkspace = useOperatingWorkspace() + let effectiveWorkspace = $derived(workspaceId ?? $operatingWorkspace) + let usesLocal = $derived( + effectiveWorkspace != undefined && effectiveWorkspace !== $workspaceStore + ) let localWorkerTags = $state(undefined) let currentTags = $derived(usesLocal ? localWorkerTags : $workerTags) diff --git a/frontend/src/lib/components/WorkerTagSelect.svelte b/frontend/src/lib/components/WorkerTagSelect.svelte index fe00f07f9d..503c8f7f66 100644 --- a/frontend/src/lib/components/WorkerTagSelect.svelte +++ b/frontend/src/lib/components/WorkerTagSelect.svelte @@ -1,5 +1,6 @@ {#snippet startSnippet({ item })} - {#if tagsToWorkerExists} + + {#if tagsToWorkerExists && !item.__is_create && item.value in tagsToWorkerExists} {#if tagsToWorkerExists[item.value]} {#snippet text()} @@ -160,7 +167,7 @@ {/if} {/if} {/snippet} -
+
{#if !noLabel}
{placeholder ?? 'tag'}
{/if} @@ -174,22 +181,30 @@ placeholder={nullTag ? nullTag : (placeholder ?? 'lang default')} items={safeSelectItems(items)} bind:value={() => tag, (value) => ((tag = value), dispatch('change', value))} + onCreateItem={(value) => ((tag = value), dispatch('change', value))} + createText="Press Enter to use this tag" {startSnippet} bottomSnippet={refreshAll} />
{#snippet refreshAll()} - +
+ + $workspace, $args[…] and $flow_expr[…] are filled in when + the job runs, and must land on an allowed tag + + +
{/snippet} diff --git a/frontend/src/lib/components/WorkflowTimeline.svelte b/frontend/src/lib/components/WorkflowTimeline.svelte index 2ea5ebba50..868f215c20 100644 --- a/frontend/src/lib/components/WorkflowTimeline.svelte +++ b/frontend/src/lib/components/WorkflowTimeline.svelte @@ -9,11 +9,19 @@ import ObjectViewer from './propertyPicker/ObjectViewer.svelte' import { CheckCircle2, XCircle } from 'lucide-svelte' import { JobService, type Job, type WorkflowStatus } from '$lib/gen' - import { enterpriseLicense, userStore, workspaceStore } from '$lib/stores' + import { enterpriseLicense } from '$lib/stores' import { Button } from '$lib/components/common' import { Alert } from '$lib/components/common' import SchemaForm from '$lib/components/SchemaForm.svelte' import { sendUserToast } from '$lib/toast' + import { + useOperatingUser, + useOperatingWorkspace + } from '$lib/components/operatingWorkspace.svelte' + + const operatingWorkspace = useOperatingWorkspace() + const operatingUser = useOperatingUser() + const actingUser = $derived(operatingUser.current) interface Props { flow_status: Record @@ -97,7 +105,7 @@ } async function fetchChildJob(id: string) { - const ws = $workspaceStore + const ws = $operatingWorkspace if (!ws) return loadingJobs[id] = true try { @@ -126,7 +134,7 @@ let approvalFormArgs: Record> = $state({}) async function handleApprove(key: string, formSchema: any) { - const ws = $workspaceStore + const ws = $operatingWorkspace if (!ws || !jobId) return approvalLoading[key] = true try { @@ -147,7 +155,7 @@ let cancelLoading = $state(false) async function handleCancel() { - const ws = $workspaceStore + const ws = $operatingWorkspace if (!ws || !jobId) return cancelLoading = true try { @@ -260,7 +268,7 @@ {msToSec(v.duration_ms ?? 0)}s {/if}
- {#if canApprove && selfApprovalDisabled && $userStore?.is_admin} + {#if canApprove && selfApprovalDisabled && actingUser?.is_admin}
Self-approval is disabled but allowed because you are an admin/owner
diff --git a/frontend/src/lib/components/WorkspaceItemDrillPicker.svelte b/frontend/src/lib/components/WorkspaceItemDrillPicker.svelte index e321679a9b..ec285ce4dd 100644 --- a/frontend/src/lib/components/WorkspaceItemDrillPicker.svelte +++ b/frontend/src/lib/components/WorkspaceItemDrillPicker.svelte @@ -14,7 +14,6 @@ standalone editor autosaves and surfacing those in the breadcrumb picker would be surprising. --> {#if !onLatest} - - By deploying, you may overwrite changes made by other users. Press 'Deploy' to see diff. + + {headUnknown + ? "Deploying may overwrite changes made by other users. Press 'Deploy' to see diff." + : "By deploying, you may overwrite changes made by other users. Press 'Deploy' to see diff."}
{/if} @@ -596,7 +613,7 @@ {/if}
- {#if !($userStore?.is_admin || $userStore?.is_super_admin)} + {#if !(actingUser?.is_admin || actingUser?.is_super_admin)} Custom path can only be set by workspace admins @@ -617,7 +634,7 @@ options={{ right: 'Use a custom URL' }} - disabled={!$enterpriseLicense || !($userStore?.is_admin || $userStore?.is_super_admin)} + disabled={!$enterpriseLicense || !(actingUser?.is_admin || actingUser?.is_super_admin)} /> {#if customPath !== undefined} @@ -625,7 +642,7 @@
Custom path
@@ -192,7 +194,7 @@
@@ -219,7 +221,7 @@ {#if job != undefined && 'result' in job && job?.result != undefined}
diff --git a/frontend/src/lib/components/apps/editor/DeploymentHistory.svelte b/frontend/src/lib/components/apps/editor/DeploymentHistory.svelte index 614d1f4fa4..63bc17a8eb 100644 --- a/frontend/src/lib/components/apps/editor/DeploymentHistory.svelte +++ b/frontend/src/lib/components/apps/editor/DeploymentHistory.svelte @@ -14,12 +14,14 @@ } from '$lib/utils' import { orderedYamlStringify } from '$lib/utils/orderedYaml' import { AppService, type AppWithLastVersion, type AppHistory } from '$lib/gen' - import { workspaceStore } from '$lib/stores' import { Skeleton } from '$lib/components/common' import Button from '$lib/components/common/button/Button.svelte' import { createEventDispatcher, untrack } from 'svelte' import { Pencil, ArrowRight, X, Loader2 } from 'lucide-svelte' import Select from '$lib/components/select/Select.svelte' + import { useOperatingWorkspace } from '$lib/components/operatingWorkspace.svelte' + + const operatingWorkspace = useOperatingWorkspace() interface Props { appPath: string | undefined @@ -48,7 +50,7 @@ return cached } - const app = await AppService.getAppByVersion({ workspace: $workspaceStore!, id: version }) + const app = await AppService.getAppByVersion({ workspace: $operatingWorkspace!, id: version }) versionCache[version] = app return app } @@ -60,7 +62,7 @@ loading = true versions = await AppService.getAppHistoryByPath({ - workspace: $workspaceStore!, + workspace: $operatingWorkspace!, path: appPath }) loading = false @@ -90,7 +92,7 @@ return } await AppService.updateAppHistory({ - workspace: $workspaceStore!, + workspace: $operatingWorkspace!, id: appId, version: appVersion, requestBody: { @@ -107,7 +109,9 @@ } function toVersionLabel(version: AppHistory): string { - return emptyString(version.deployment_msg) ? `Version ${version.version}` : version.deployment_msg! + return emptyString(version.deployment_msg) + ? `Version ${version.version}` + : version.deployment_msg! } let availableVersions = $derived( diff --git a/frontend/src/lib/components/apps/editor/appDeploy.svelte.ts b/frontend/src/lib/components/apps/editor/appDeploy.svelte.ts index 122c42bcf9..f799cb864a 100644 --- a/frontend/src/lib/components/apps/editor/appDeploy.svelte.ts +++ b/frontend/src/lib/components/apps/editor/appDeploy.svelte.ts @@ -1,7 +1,7 @@ -import { base } from "$lib/base" -import { workspaceStore } from "$lib/stores" -import { get } from "svelte/store" +import { base } from '$lib/base' +import { workspaceStore } from '$lib/stores' +import { get } from 'svelte/store' export function computeSecretUrl(secretUrl: string) { - return `${window.location.origin}${base}/public/${get(workspaceStore)}/${secretUrl}` + return `${window.location.origin}${base}/public/${get(workspaceStore)}/${secretUrl}` } diff --git a/frontend/src/lib/components/apps/editor/inlineScriptsPanel/InlineScriptRunnableByPath.svelte b/frontend/src/lib/components/apps/editor/inlineScriptsPanel/InlineScriptRunnableByPath.svelte index 3089809f12..293d83b4ae 100644 --- a/frontend/src/lib/components/apps/editor/inlineScriptsPanel/InlineScriptRunnableByPath.svelte +++ b/frontend/src/lib/components/apps/editor/inlineScriptsPanel/InlineScriptRunnableByPath.svelte @@ -19,7 +19,6 @@ CtxAppInput } from '../../inputType' import type { AppViewerContext } from '../../types' - import { workspaceStore } from '$lib/stores' import { createEventDispatcher } from 'svelte' import { deepEqual } from 'fast-equals' import { computeFields } from './utils' @@ -35,6 +34,9 @@ import FlowEditorDrawer from '$lib/components/flows/content/FlowEditorDrawer.svelte' import { FlowService, ScriptService, type OpenFlow } from '$lib/gen' import { replaceScriptPlaceholderWithItsValues } from '$lib/hub' + import { useOperatingWorkspace } from '$lib/components/operatingWorkspace.svelte' + + const operatingWorkspace = useOperatingWorkspace() interface Props { runnable: RunnableByPath @@ -131,7 +133,7 @@ return } - const loaded = await loadSchema($workspaceStore ?? '', runnable.path, 'flow') + const loaded = await loadSchema($operatingWorkspace ?? '', runnable.path, 'flow') const schema = loaded?.schema ?? emptySchema() if (!deepEqual(runnable.schema, schema)) { runnable.schema = schema @@ -167,7 +169,7 @@ async function openScriptEditor(path: string) { try { const script = await ScriptService.getScriptByPath({ - workspace: $workspaceStore!, + workspace: $operatingWorkspace!, path }) scriptEditorDrawer?.openDrawer(script.hash, () => { @@ -344,7 +346,7 @@ startIcon={{ icon: Eye }} endIcon={{ icon: ExternalLink }} target="_blank" - href="{base}/flows/get/{runnable.path}?workspace={$workspaceStore}" + href="{base}/flows/get/{runnable.path}?workspace={$operatingWorkspace}" > Details @@ -425,7 +427,7 @@ {#if runnable.runType == 'flow' && isHubFlowPath(runnable.path)} Hub flow not found at {runnable.path} {:else} - {runnable.runType} not found at {runnable.path} in workspace {$workspaceStore} + {runnable.runType} not found at {runnable.path} in workspace {$operatingWorkspace} {/if}
{:else if runnable.runType == 'script' || runnable.runType == 'hubscript'} diff --git a/frontend/src/lib/components/apps/editor/settingsPanel/mainInput/RunnableSelector.svelte b/frontend/src/lib/components/apps/editor/settingsPanel/mainInput/RunnableSelector.svelte index 8543fc0683..08b8ac6219 100644 --- a/frontend/src/lib/components/apps/editor/settingsPanel/mainInput/RunnableSelector.svelte +++ b/frontend/src/lib/components/apps/editor/settingsPanel/mainInput/RunnableSelector.svelte @@ -11,15 +11,12 @@ import type { Schema } from '$lib/common' import { emptySchema } from '$lib/utils' import { loadSchema } from '$lib/infer' - import { workspaceStore } from '$lib/stores' + import { useOperatingWorkspace } from '$lib/components/operatingWorkspace.svelte' + + const operatingWorkspace = useOperatingWorkspace() import { buildPathRunnableSelection } from './runnableSelectorUtils' - type TabType = - | 'hubscripts' - | 'hubflows' - | 'workspacescripts' - | 'workspaceflows' - | 'inlinescripts' + type TabType = 'hubscripts' | 'hubflows' | 'workspacescripts' | 'workspaceflows' | 'inlinescripts' interface Props { defaultUserInput?: boolean @@ -60,7 +57,7 @@ path: string, runType: 'script' | 'flow' | 'hubscript' ): Promise<{ schema: Schema; summary: string | undefined }> { - const schema = await loadSchema($workspaceStore!, path, runType) + const schema = await loadSchema($operatingWorkspace!, path, runType) if (!schema.schema.order) { schema.schema.order = Object.keys(schema.schema.properties ?? {}) } diff --git a/frontend/src/lib/components/apps/editor/settingsPanel/mainInput/WorkspaceFlowList.svelte b/frontend/src/lib/components/apps/editor/settingsPanel/mainInput/WorkspaceFlowList.svelte index b2cc91fc01..7831e83458 100644 --- a/frontend/src/lib/components/apps/editor/settingsPanel/mainInput/WorkspaceFlowList.svelte +++ b/frontend/src/lib/components/apps/editor/settingsPanel/mainInput/WorkspaceFlowList.svelte @@ -7,9 +7,11 @@ import NoItemFound from '$lib/components/home/NoItemFound.svelte' import RowIcon from '$lib/components/common/table/RowIcon.svelte' import { FlowService, type Flow } from '$lib/gen' - import { workspaceStore } from '$lib/stores' import { emptyString } from '$lib/utils' import { Skeleton } from '$lib/components/common' + import { useOperatingWorkspace } from '$lib/components/operatingWorkspace.svelte' + + const operatingWorkspace = useOperatingWorkspace() interface Props { filter?: string @@ -26,7 +28,7 @@ async function loadFlow(): Promise { const loadedFlows = await FlowService.listFlows({ - workspace: $workspaceStore!, + workspace: $operatingWorkspace!, perPage: 300, withoutDescription: true }) diff --git a/frontend/src/lib/components/apps/editor/settingsPanel/mainInput/WorkspaceScriptList.svelte b/frontend/src/lib/components/apps/editor/settingsPanel/mainInput/WorkspaceScriptList.svelte index be261ca5e8..13447c4787 100644 --- a/frontend/src/lib/components/apps/editor/settingsPanel/mainInput/WorkspaceScriptList.svelte +++ b/frontend/src/lib/components/apps/editor/settingsPanel/mainInput/WorkspaceScriptList.svelte @@ -7,9 +7,11 @@ import NoItemFound from '$lib/components/home/NoItemFound.svelte' import RowIcon from '$lib/components/common/table/RowIcon.svelte' import { type Script, ScriptService } from '$lib/gen' - import { workspaceStore } from '$lib/stores' import { emptyString } from '$lib/utils' import { Skeleton } from '$lib/components/common' + import { useOperatingWorkspace } from '$lib/components/operatingWorkspace.svelte' + + const operatingWorkspace = useOperatingWorkspace() interface Props { filter?: string @@ -26,7 +28,7 @@ async function loadScripts(): Promise { const loadedScripts = await ScriptService.listScripts({ - workspace: $workspaceStore!, + workspace: $operatingWorkspace!, perPage: 300, withoutDescription: true }) diff --git a/frontend/src/lib/components/apps/types.ts b/frontend/src/lib/components/apps/types.ts index 960e273ac3..75de2d7faf 100644 --- a/frontend/src/lib/components/apps/types.ts +++ b/frontend/src/lib/components/apps/types.ts @@ -183,6 +183,11 @@ export interface AppEditorProps { // AppEditorHeader as a callback prop rather than `on:restore` forwarding, // which does not propagate through these runes-mode components. onRestore?: (restoredApp: any) => void + // Fired after a successful deploy, which keeps this editor open: `version` is what + // the deploy wrote, for the next draft's fork base, and `head` what is deployed now, + // with its author and time. `version` and `head` differ when another deploy landed + // beside this one, which is when the out-of-date prompt has something to say. + onDeploy?: (e: { version?: number; head?: number; headBy?: string; headAt?: string }) => void } export type App = { diff --git a/frontend/src/lib/components/assets/AssetGraph/AssetGraphDetailsPane.svelte b/frontend/src/lib/components/assets/AssetGraph/AssetGraphDetailsPane.svelte index 4b252b9338..6050c2d1cd 100644 --- a/frontend/src/lib/components/assets/AssetGraph/AssetGraphDetailsPane.svelte +++ b/frontend/src/lib/components/assets/AssetGraph/AssetGraphDetailsPane.svelte @@ -48,7 +48,10 @@ import AssetRunsPanel from './AssetRunsPanel.svelte' import { Pane, Splitpanes } from 'svelte-splitpanes' import { fade } from 'svelte/transition' - import { userStore } from '$lib/stores' + import { useOperatingUser } from '$lib/components/operatingWorkspace.svelte' + + const operatingUser = useOperatingUser() + const actingUser = $derived(operatingUser.current) interface Props { // Regular selection — loads the script by path for inline editing. @@ -662,7 +665,7 @@ // hand-rolled here because we need *two* confirm buttons, not one. let removeOpen = $state(false) let removing = $state(false) - let canHardDelete = $derived(!!($userStore?.is_admin || $userStore?.is_super_admin)) + let canHardDelete = $derived(!!(actingUser?.is_admin || actingUser?.is_super_admin)) // React to the parent's remove-signal counter and pop the same modal // the in-pane trash button uses. Skipped for drafts (the parent calls diff --git a/frontend/src/lib/components/assets/AssetGraph/AssetNode.svelte b/frontend/src/lib/components/assets/AssetGraph/AssetNode.svelte index 815db36b18..e719bfe2bd 100644 --- a/frontend/src/lib/components/assets/AssetGraph/AssetNode.svelte +++ b/frontend/src/lib/components/assets/AssetGraph/AssetNode.svelte @@ -19,12 +19,15 @@ XCircle } from 'lucide-svelte' import type { ScriptLang } from '$lib/gen' - import { enterpriseLicense, workspaceStore } from '$lib/stores' + import { enterpriseLicense } from '$lib/stores' import { sendUserToast } from '$lib/utils' import { PIPELINE_LANGUAGES } from './pipelineLanguages' import type { PipelineOutputKind } from './pipelineTemplates' import type { DbtAssetProvenance } from './types' import DbtIcon from '$lib/components/icons/DbtIcon.svelte' + import { useOperatingWorkspace } from '$lib/components/operatingWorkspace.svelte' + + const operatingWorkspace = useOperatingWorkspace() // Shape used for both the data prop and the run callback. Drafts carry // `content` / `language` so the page-level run handler can dispatch to @@ -118,7 +121,7 @@ async function runProducers(e: MouseEvent) { e.stopPropagation() - if (!$workspaceStore || running || !data.onRunProducer) return + if (!$operatingWorkspace || running || !data.onRunProducer) return if (scriptProducers.length === 0) return running = true const handler = data.onRunProducer @@ -170,14 +173,14 @@ } const cols = Object.entries(d.columns ?? {}) if (cols.length) { - lines.push( - `columns: ${cols.map(([c, desc]) => (desc ? `${c} (${desc})` : c)).join(', ')}` - ) + lines.push(`columns: ${cols.map(([c, desc]) => (desc ? `${c} (${desc})` : c)).join(', ')}`) } if (d.freshness) { const f = d.freshness as Record const window = (k: string) => - f[k]?.count != null ? `${k.replace('_after', '')} after ${f[k].count}${f[k].period?.[0] ?? ''}` : '' + f[k]?.count != null + ? `${k.replace('_after', '')} after ${f[k].count}${f[k].period?.[0] ?? ''}` + : '' const windows = ['warn_after', 'error_after'].map(window).filter(Boolean) if (windows.length) lines.push(`freshness: ${windows.join(', ')}`) } diff --git a/frontend/src/lib/components/assets/AssetGraph/AssetRunsPanel.svelte b/frontend/src/lib/components/assets/AssetGraph/AssetRunsPanel.svelte index f43259b561..7cee1deb76 100644 --- a/frontend/src/lib/components/assets/AssetGraph/AssetRunsPanel.svelte +++ b/frontend/src/lib/components/assets/AssetGraph/AssetRunsPanel.svelte @@ -8,7 +8,6 @@ // seconds so the user sees status transitions without refreshing — // JobLoader handles streaming for the *selected* job. import { JobService, type Job } from '$lib/gen' - import { workspaceStore } from '$lib/stores' import { onDestroy, untrack } from 'svelte' import { displayDate } from '$lib/utils' import { CheckCircle2, Clock, History, Loader2, XCircle, Ban } from 'lucide-svelte' @@ -21,6 +20,9 @@ import { Popover } from '$lib/components/meltComponents' import { twMerge } from 'tailwind-merge' import DispatchEventsButton from '$lib/components/runs/DispatchEventsButton.svelte' + import { useOperatingWorkspace } from '$lib/components/operatingWorkspace.svelte' + + const operatingWorkspace = useOperatingWorkspace() interface Props { // Producers of this asset. Each contributes its own listExtendedJobs @@ -88,7 +90,7 @@ // Run only when the *content* of the producer set changes (string // key) or when the parent explicitly bumps `refreshKey` after // dispatching a new run. The producerKey debounce alone isn't - // enough: refresh() synchronously reads $workspaceStore and + // enough: refresh() synchronously reads $operatingWorkspace and // runnableProducers before the first await, and Svelte 5 records // those as deps of the surrounding $effect — which would then // re-fire whenever the parent re-derives `producers` (i.e. on @@ -174,11 +176,11 @@ async function refresh(): Promise { if (refreshInFlight) return refreshInFlight - if (!$workspaceStore || runnableProducers.length === 0) { + if (!$operatingWorkspace || runnableProducers.length === 0) { jobs = [] return } - const ws = $workspaceStore + const ws = $operatingWorkspace // Capture the producer paths *now* — using runnableProducers // directly inside the await would re-read after the array // identity churned, defeating the in-flight guard. @@ -286,12 +288,12 @@ Open ↗ - {#if $workspaceStore} + {#if $operatingWorkspace} {/if} @@ -397,7 +399,7 @@
diff --git a/frontend/src/lib/components/assets/AssetGraph/DataTablePreview.svelte b/frontend/src/lib/components/assets/AssetGraph/DataTablePreview.svelte index 1b86fa6853..5f5dc98b60 100644 --- a/frontend/src/lib/components/assets/AssetGraph/DataTablePreview.svelte +++ b/frontend/src/lib/components/assets/AssetGraph/DataTablePreview.svelte @@ -9,7 +9,6 @@ // just be noise. import DBTable from '$lib/components/DBTable.svelte' import { resource } from 'runed' - import { workspaceStore } from '$lib/stores' import { loadAllTablesMetaData } from '$lib/components/apps/components/display/dbtable/metadata' import { dbTableOpsWithPreviewScripts } from '$lib/components/dbOps' import { WorkspaceService } from '$lib/gen' @@ -19,6 +18,9 @@ import Button from '$lib/components/common/button/Button.svelte' import { base } from '$lib/base' import { twMerge } from 'tailwind-merge' + import { useOperatingWorkspace } from '$lib/components/operatingWorkspace.svelte' + + const operatingWorkspace = useOperatingWorkspace() interface Props { // Asset path as parsed from `datatable:///
` — @@ -71,7 +73,7 @@ // query, and the user needs to configure it before pipeline scripts // can write to it. let datatables = resource( - () => $workspaceStore, + () => $operatingWorkspace, async (ws) => { if (!ws) return [] as string[] try { @@ -117,9 +119,9 @@ () => [input, refreshKey], async ([_input]) => { colDefsError = undefined - if (!_input || !$workspaceStore) return undefined + if (!_input || !$operatingWorkspace) return undefined try { - return await loadAllTablesMetaData($workspaceStore, _input) + return await loadAllTablesMetaData($operatingWorkspace, _input) } catch (e) { colDefsError = (e as Error)?.message || String(e) return undefined @@ -138,12 +140,12 @@ }) let dbTableOps = $derived( - input && tableColDefs && parsed.table && $workspaceStore + input && tableColDefs && parsed.table && $operatingWorkspace ? dbTableOpsWithPreviewScripts({ input, tableKey: parsed.table, colDefs: tableColDefs, - workspace: $workspaceStore + workspace: $operatingWorkspace }) : undefined ) diff --git a/frontend/src/lib/components/assets/AssetGraph/DucklakeResultPreview.svelte b/frontend/src/lib/components/assets/AssetGraph/DucklakeResultPreview.svelte index f21ca486af..d2d3c03b8a 100644 --- a/frontend/src/lib/components/assets/AssetGraph/DucklakeResultPreview.svelte +++ b/frontend/src/lib/components/assets/AssetGraph/DucklakeResultPreview.svelte @@ -9,7 +9,6 @@ // ("This partition") or show the full table ("Whole table") via a toggle. import DBTable from '$lib/components/DBTable.svelte' import { resource } from 'runed' - import { workspaceStore } from '$lib/stores' import { loadAllTablesMetaData } from '$lib/components/apps/components/display/dbtable/metadata' import { dbTableOpsWithPreviewScripts } from '$lib/components/dbOps' import type { DbInput } from '$lib/components/dbTypes' @@ -18,6 +17,9 @@ import { twMerge } from 'tailwind-merge' import ToggleButtonGroup from '$lib/components/common/toggleButton-v2/ToggleButtonGroup.svelte' import ToggleButton from '$lib/components/common/toggleButton-v2/ToggleButton.svelte' + import { useOperatingWorkspace } from '$lib/components/operatingWorkspace.svelte' + + const operatingWorkspace = useOperatingWorkspace() interface Props { // Full asset URI, e.g. `ducklake://main/orders_daily`. @@ -61,9 +63,9 @@ () => [input, refreshKey] as const, async ([_input]) => { colDefsError = undefined - if (!_input || !$workspaceStore) return undefined + if (!_input || !$operatingWorkspace) return undefined try { - return await loadAllTablesMetaData($workspaceStore, _input) + return await loadAllTablesMetaData($operatingWorkspace, _input) } catch (e) { colDefsError = (e as Error)?.message || String(e) return undefined @@ -89,12 +91,12 @@ }) let dbTableOps = $derived.by(() => { - if (!(input && tableColDefs && tableKey && $workspaceStore)) return undefined + if (!(input && tableColDefs && tableKey && $operatingWorkspace)) return undefined const ops = dbTableOpsWithPreviewScripts({ input, tableKey, colDefs: tableColDefs, - workspace: $workspaceStore, + workspace: $operatingWorkspace, whereClause }) // Read-only preview: drop the mutation handlers so DBTable hides its diff --git a/frontend/src/lib/components/assets/AssetGraph/DucklakeVersionPreview.svelte b/frontend/src/lib/components/assets/AssetGraph/DucklakeVersionPreview.svelte index 5afb51fe66..35eff3beae 100644 --- a/frontend/src/lib/components/assets/AssetGraph/DucklakeVersionPreview.svelte +++ b/frontend/src/lib/components/assets/AssetGraph/DucklakeVersionPreview.svelte @@ -7,7 +7,6 @@ // hand is shown above the grid so the affordance is self-documenting. import DBTable from '$lib/components/DBTable.svelte' import { resource } from 'runed' - import { workspaceStore } from '$lib/stores' import { fetchDucklakeColumnsAtVersion, dbTableOpsWithPreviewScripts @@ -17,6 +16,9 @@ import { AlertTriangle, Loader2, ClipboardCopy } from 'lucide-svelte' import { Button } from '$lib/components/common' import { twMerge } from 'tailwind-merge' + import { useOperatingWorkspace } from '$lib/components/operatingWorkspace.svelte' + + const operatingWorkspace = useOperatingWorkspace() interface Props { // Full asset URI, e.g. `ducklake://main/orders_daily`. @@ -56,9 +58,10 @@ let columns = resource( () => [ducklake, tableKey, version] as const, async ([_ducklake, _tableKey, _version]) => { - if (!_ducklake || !_tableKey || _version == undefined || !$workspaceStore) return undefined + if (!_ducklake || !_tableKey || _version == undefined || !$operatingWorkspace) + return undefined const colDefs = await fetchDucklakeColumnsAtVersion({ - workspace: $workspaceStore, + workspace: $operatingWorkspace, ducklake: _ducklake, tableKey: _tableKey, version: _version @@ -73,13 +76,13 @@ let tableColDefs = $derived(ready ? columns.current!.colDefs : undefined) let dbTableOps = $derived.by(() => { - if (!(input && tableColDefs && tableKey && $workspaceStore && version != undefined)) + if (!(input && tableColDefs && tableKey && $operatingWorkspace && version != undefined)) return undefined const ops = dbTableOpsWithPreviewScripts({ input, tableKey, colDefs: tableColDefs, - workspace: $workspaceStore, + workspace: $operatingWorkspace, version }) // Historical reads are immutable: drop every mutation handler so DBTable diff --git a/frontend/src/lib/components/assets/AssetGraph/PipelineScriptView.svelte b/frontend/src/lib/components/assets/AssetGraph/PipelineScriptView.svelte index 79a1cf6d47..2450595b6d 100644 --- a/frontend/src/lib/components/assets/AssetGraph/PipelineScriptView.svelte +++ b/frontend/src/lib/components/assets/AssetGraph/PipelineScriptView.svelte @@ -6,7 +6,9 @@ import PipelineRunForm from './PipelineRunForm.svelte' import AssetRunsPanel from './AssetRunsPanel.svelte' import { Pane, Splitpanes } from 'svelte-splitpanes' - import { workspaceStore } from '$lib/stores' + import { useOperatingWorkspace } from '$lib/components/operatingWorkspace.svelte' + + const operatingWorkspace = useOperatingWorkspace() import { parsePipelineAnnotations } from './parsePipelineAnnotations' interface Props { @@ -201,7 +203,7 @@ bind:args bind:isValid {partitionSpec} - workspace={$workspaceStore ?? ''} + workspace={$operatingWorkspace ?? ''} {materializeTarget} {upstreamAssets} /> diff --git a/frontend/src/lib/components/assets/AssetGraph/PipelineTriggerEditors.svelte b/frontend/src/lib/components/assets/AssetGraph/PipelineTriggerEditors.svelte index e364b29883..4f97039a1c 100644 --- a/frontend/src/lib/components/assets/AssetGraph/PipelineTriggerEditors.svelte +++ b/frontend/src/lib/components/assets/AssetGraph/PipelineTriggerEditors.svelte @@ -1,5 +1,4 @@ { open = false @@ -33,7 +44,11 @@ on:confirmed={() => confirmCallback()} >
- A new version was deployed while you were editing this one. + + {headUnknown + ? 'This editor could not check whether a newer version is deployed, so it cannot tell whether this overwrites newer work.' + : 'A new version was deployed while you were editing this one.'} + {#if diffDrawer} diff --git a/frontend/src/lib/components/common/confirmationModal/DraftChangesConfirmationModal.svelte b/frontend/src/lib/components/common/confirmationModal/DraftChangesConfirmationModal.svelte index dadf814a80..5a51b4cfba 100644 --- a/frontend/src/lib/components/common/confirmationModal/DraftChangesConfirmationModal.svelte +++ b/frontend/src/lib/components/common/confirmationModal/DraftChangesConfirmationModal.svelte @@ -9,9 +9,12 @@ import { Bot, Star } from 'lucide-svelte' import ToggleButtonGroup from '../toggleButton-v2/ToggleButtonGroup.svelte' import ToggleButton from '../toggleButton-v2/ToggleButton.svelte' - import { userStore } from '$lib/stores' import Badge from '../badge/Badge.svelte' import type { LinkedAgentDraft } from '$lib/components/flows/linkedAgentDrafts' + import { useOperatingUser } from '$lib/components/operatingWorkspace.svelte' + + const operatingUser = useOperatingUser() + const actingUser = $derived(operatingUser.current) interface Props { open?: boolean @@ -80,8 +83,8 @@ // Creating http trigger is forbidden for non-admin users const adminOnly = trigger.type === 'http' && - !$userStore?.is_admin && - !$userStore?.is_super_admin && + !actingUser?.is_admin && + !actingUser?.is_super_admin && trigger.isDraft const invalidConfig = !trigger.draftConfig?.canSave diff --git a/frontend/src/lib/components/common/confirmationModal/DraftEditorModals.svelte b/frontend/src/lib/components/common/confirmationModal/DraftEditorModals.svelte index 26366325b5..20c29286b5 100644 --- a/frontend/src/lib/components/common/confirmationModal/DraftEditorModals.svelte +++ b/frontend/src/lib/components/common/confirmationModal/DraftEditorModals.svelte @@ -13,6 +13,10 @@ import type { UserDraftItemKind } from '$lib/gen' import DraftSyncConflictModal from './DraftSyncConflictModal.svelte' import OtherUsersDraftsModal, { type OtherDraftUser } from './OtherUsersDraftsModal.svelte' + import { UserDraftDbSyncer } from '$lib/userDraftDbSyncer.svelte' + import { sendUserToast } from '$lib/toast' + import { base } from '$app/paths' + import { goto } from '$app/navigation' import StaleDraftModal from './StaleDraftModal.svelte' import ConfirmationModal from './ConfirmationModal.svelte' import { OtherUserDraftLoad } from '$lib/components/otherUserDraftLoad.svelte' @@ -38,15 +42,27 @@ draftSavedAt?: string | undefined /** ISO timestamp of the latest deploy at this path. */ deployedAt?: string | undefined - /** Precise staleness inputs (flows/apps): the deployed version the draft was - * forked from, and the current deployed head. When both are set they drive - * `isStale` and the dedup key instead of the timestamps — exact, and stable - * across autosaves (the timestamp drifts past `deployedAt` as you keep - * editing). Absent (pre-feature drafts, scripts) ⇒ timestamp fallback. */ - draftBaseVersion?: number | undefined - deployedHeadVersion?: number | undefined + /** Precise staleness inputs: the deployed version the draft forked from + * (`draft_base` on the get-by-path response) and the current deployed head, + * both as text whatever the kind. When both are set they drive `isStale` and + * the dedup key instead of the timestamps, which drift past `deployedAt` as + * you keep editing. Absent (a draft never forked from a deploy) ⇒ timestamp + * fallback. */ + draftBaseVersion?: string | undefined + deployedHeadVersion?: string | undefined + /** Who deployed the head, named in the stale prompt. */ + deployedBy?: string | undefined /** Discard the draft and reload deployed (same as "Reset to deployed"). */ onLoadLatestDeploy?: () => void | Promise + /** Opens the editor's Deployed↔Current diff from the stale prompt, so the + * choice between keeping and discarding is informed. Omit where the editor + * has no diff drawer; the action is then not rendered. */ + onViewDiff?: () => void | Promise + /** Runs before this editor follows its draft to the item's new path: the + * editor's own draft save, which materializes text the code editor still + * holds. Without it, keystrokes typed since the relocating save are lost + * to the navigation. */ + onBeforeRelocate?: () => void | Promise /** Defaults to true; set to false to suppress all modals. */ enabled?: boolean } @@ -65,7 +81,10 @@ deployedAt = undefined, draftBaseVersion = undefined, deployedHeadVersion = undefined, + deployedBy = undefined, onLoadLatestDeploy, + onViewDiff, + onBeforeRelocate, enabled = true }: Props = $props() @@ -74,9 +93,9 @@ let staleAlertKey = $state(undefined) let staleModalOpen = $state(false) - // Prefer the exact version comparison (flows/apps) over the timestamp: the - // draft's pinned fork base never drifts, whereas `draftSavedAt` advances past - // `deployedAt` once you keep editing a stale draft, hiding the staleness. + // Prefer the version comparison over the timestamp for every kind that supplies + // one: `draftSavedAt` advances past `deployedAt` as you keep editing, hiding the + // staleness outright. const useVersion = $derived(draftBaseVersion != null && deployedHeadVersion != null) const isStale = $derived( !!onLoadLatestDeploy && @@ -105,6 +124,48 @@ } }) }) + + const EDITOR_SEGMENT: Partial> = { + script: 'scripts/edit', + flow: 'flows/edit', + app: 'apps/edit', + raw_app: 'apps_raw/edit' + } + + // The item was moved while this editor was open: the draft row followed it + // and the save just landed there. Follow it too — the route reloads the item + // at its new path, and the stale prompt above then says what changed. Edits + // typed since that save are flushed first, so leaving this path drops none. + $effect(() => { + if (!enabled || !workspace || !path) return + const seg = EDITOR_SEGMENT[itemKind] + if (!seg) return + const query = { workspace, itemKind, path } + // The flush below saves again and can land here a second time, and a second move + // can land while it runs: the last destination reported is the one to follow. + let relocating = false + let destination: string | undefined = undefined + return UserDraftDbSyncer.onRelocated(query, async (newPath) => { + destination = newPath + if (relocating) return + relocating = true + await onBeforeRelocate?.() + await UserDraftDbSyncer.flush(query) + // `flush` resolves on a failed or rejected save as well, and leaving the + // route drops what it was carrying: stay, so the editor keeps the edits + // and its own failure indicator. + if ( + UserDraftDbSyncer.getState(query).failureMessage || + UserDraftDbSyncer.getConflict(query).conflict + ) { + relocating = false + return + } + const target = destination ?? newPath + sendUserToast(`This item was moved to ${target}. You are now editing it there.`) + await goto(`${base}/${seg}/${target}`) + }) + }) {#if enabled && workspace && path} @@ -130,9 +191,14 @@ {#if onLoadLatestDeploy} {/if} /** - * Modal opened on editor mount when the authed user's per-user draft - * is older than the latest deployed version at the same path — i.e. - * someone else deployed a new version while the draft was sitting - * around. The user is asked to either pick up the latest deploy - * (discards the stale draft) or keep editing what they had. - * - * The parent threads `draftSavedAt` and `deployedAt` raw and the - * modal computes staleness internally; this keeps each route from - * re-implementing the comparison and the threshold (we treat a - * draft as stale only when it's strictly older — a deploy at the - * exact same instant is treated as not stale). + * The prompt for a draft that is behind: someone deployed a newer version + * of the item after the draft forked from it. Opened on every load while + * that holds (the parent computes it; see DraftEditorModals), it names the + * two versions and offers two ways out: look at the diff, or drop the draft + * for the latest deploy. Closing it is "keep editing". Taking the latest as + * the new base while keeping the edits lives in the diff drawer, where the + * user can see what they are taking. * * Open-state is bindable so the parent can dismiss programmatically * (e.g. after the load-latest-deploy callback completes). */ import Modal2 from '$lib/components/common/modal/Modal2.svelte' import Button from '$lib/components/common/button/Button.svelte' - import { AlertTriangle } from 'lucide-svelte' + import { AlertTriangle, GitCompare } from 'lucide-svelte' import { sendUserToast } from '$lib/toast' + import type { UserDraftItemKind } from '$lib/gen' type Props = { isOpen: boolean - /** ISO timestamp the authed user's draft was saved. */ + itemKind: UserDraftItemKind + /** ISO timestamp the authed user's draft was saved. Shown only when the + * versions below are unknown (a draft that predates the base). */ draftSavedAt: string | undefined /** ISO timestamp the latest deploy at this path landed. */ deployedAt: string | undefined + /** The version the draft forked from and the deployed head, as text. */ + draftBaseVersion?: string | undefined + deployedHeadVersion?: string | undefined + /** Who deployed the head. */ + deployedBy?: string | undefined /** Discards the draft and reloads the deployed payload — the route * already has this callback for the AutosaveIndicator's "Reset to * deployed" button; pass the same function in. */ onLoadLatestDeploy: () => void | Promise + /** Opens the editor's own Deployed↔Current diff. Without it the user is + * asked to choose between keeping and discarding their draft with no way + * to see what actually differs — and after a rename the difference is + * often only the path. Omitted where the editor has no diff drawer. */ + onViewDiff?: () => void | Promise } - let { isOpen = $bindable(), draftSavedAt, deployedAt, onLoadLatestDeploy }: Props = $props() + let { + isOpen = $bindable(), + itemKind, + draftSavedAt, + deployedAt, + draftBaseVersion = undefined, + deployedHeadVersion = undefined, + deployedBy = undefined, + onLoadLatestDeploy, + onViewDiff + }: Props = $props() let loading = $state(false) + // Scripts are versioned by hash, the other kinds by a numeric version id; + // the diff picker renders them the same way. + function formatVersion(v: string): string { + return itemKind === 'script' ? v.slice(0, 8) : v + } + async function loadLatestDeploy() { if (loading) return loading = true @@ -49,6 +74,13 @@ } } + // Dismisses on the way out: the diff drawer opens behind this modal, so + // leaving it up would cover the thing the user asked to see. + async function viewDiff() { + isOpen = false + await onViewDiff?.() + } + function formatTs(ts: string | undefined): string { if (!ts) return '' try { @@ -64,22 +96,38 @@
-

- A newer version was deployed after you started editing. Your draft is based on the older - deploy. -

-

- Draft saved {formatTs(draftSavedAt)} · Deployed {formatTs(deployedAt)} -

+

A newer version was deployed after you started editing.

+ {#if draftBaseVersion && deployedHeadVersion} +

+ Your draft is based on {formatVersion(draftBaseVersion)} + · latest is {formatVersion(deployedHeadVersion)} + {#if deployedBy}by {deployedBy}{/if}{#if deployedAt}, {formatTs(deployedAt)}{/if} +

+ {:else} +

+ Draft saved {formatTs(draftSavedAt)} · Deployed {formatTs(deployedAt)} +

+ {/if}
-
- - + {:else} +
+ {/if} + +
diff --git a/frontend/src/lib/components/common/drawer/DrawerContent.svelte b/frontend/src/lib/components/common/drawer/DrawerContent.svelte index ed2ed68519..9664f3a27e 100644 --- a/frontend/src/lib/components/common/drawer/DrawerContent.svelte +++ b/frontend/src/lib/components/common/drawer/DrawerContent.svelte @@ -30,6 +30,8 @@ * the content hug it with tight top padding; new entities keep normal padding. */ bannerReserved?: boolean + /** For content rendered in place of a drawer, which has nothing to close. */ + hideClose?: boolean children?: import('svelte').Snippet } @@ -50,6 +52,7 @@ titleExtra, banner, bannerReserved = false, + hideClose = false, children }: Props = $props() @@ -69,19 +72,26 @@ )} {id} > -
+
-
{ - dispatch('close') - } - }} - > - -
+ {#if !hideClose} +
{ + dispatch('close') + } + }} + > + +
+ {/if} {title ?? ''} {#if tooltip != '' || documentationLink} diff --git a/frontend/src/lib/components/common/fileDownload/FileDownload.svelte b/frontend/src/lib/components/common/fileDownload/FileDownload.svelte index 8d6529764b..c1a29fae70 100644 --- a/frontend/src/lib/components/common/fileDownload/FileDownload.svelte +++ b/frontend/src/lib/components/common/fileDownload/FileDownload.svelte @@ -1,8 +1,10 @@ + +{#snippet leafIcon(leaf: DrillLeaf)} + +{/snippet} + +{#snippet branchIcon(branch: DrillBranch)} + {#if branch.icon} + {@const Icon = branch.icon} + + {:else} + + {/if} +{/snippet} + + +
+ relativizeWorkspacePath(workspaceItemDisplayPath(leaf.data), scope)} + onScopeChange={handleScopeChange} + onFilterChange={loader.onFilterChange} + rootLoading={WORKSPACE_KINDS.some((k) => !loader.loaded[k] && loader.loadingKind[k])} + /> +
diff --git a/frontend/src/lib/components/copilot/chat/anthropic.test.ts b/frontend/src/lib/components/copilot/chat/anthropic.test.ts index 1ac413973f..9f6b49a9e9 100644 --- a/frontend/src/lib/components/copilot/chat/anthropic.test.ts +++ b/frontend/src/lib/components/copilot/chat/anthropic.test.ts @@ -1,6 +1,11 @@ import { describe, expect, it, vi } from 'vitest' import type { ChatCompletionMessageParam } from 'openai/resources/index.mjs' -import { convertOpenAIToAnthropicMessages, partialWebSearchQuery } from './anthropic' +import { + convertOpenAIToAnthropicMessages, + parseAnthropicCompletion, + partialWebSearchQuery +} from './anthropic' +import { OutputTokenLimitError } from './outputTokenLimit' // anthropic.ts pulls in the chat client/registry layer at import time; the // converter under test is pure, so stub those side-effecting modules away. @@ -217,3 +222,28 @@ describe('partialWebSearchQuery', () => { expect(partialWebSearchQuery('{"query": ')).toBeUndefined() }) }) + +describe('parseAnthropicCompletion output token limit', () => { + it('fails a message that stopped at max_tokens instead of ending the turn', async () => { + const stream = { + on: () => undefined, + done: async () => undefined, + finalMessage: async () => ({ + content: [{ type: 'thinking', thinking: 'I need to plan the flow so each' }], + stop_reason: 'max_tokens', + usage: { input_tokens: 20000, output_tokens: 64000 } + }) + } + + const parsed = parseAnthropicCompletion( + stream as any, + { onNewToken: vi.fn(), onMessageEnd: vi.fn(), setToolStatus: vi.fn() } as any, + [], + [], + [], + {} + ) + + await expect(parsed).rejects.toBeInstanceOf(OutputTokenLimitError) + }) +}) diff --git a/frontend/src/lib/components/copilot/chat/anthropic.ts b/frontend/src/lib/components/copilot/chat/anthropic.ts index 8d38f10916..122425e264 100644 --- a/frontend/src/lib/components/copilot/chat/anthropic.ts +++ b/frontend/src/lib/components/copilot/chat/anthropic.ts @@ -24,6 +24,7 @@ import { type ToolCallbacks, type WebSearchSource } from './shared' +import { OutputTokenLimitError } from './outputTokenLimit' import { anthropicUsageToChatTokenUsage, type ChatTokenUsage } from './tokenUsage' import { parseImageDataUrl } from './imageUtils' @@ -140,7 +141,8 @@ export async function getAnthropicCompletion( const { provider, config } = getProviderAndCompletionConfig({ messages, stream: true, - forceModelProvider: options?.forceModelProvider + forceModelProvider: options?.forceModelProvider, + reasoningEffort: options?.reasoningEffort }) const { system, messages: anthropicMessages } = convertOpenAIToAnthropicMessages(messages) let anthropicTools = convertOpenAIToolsToAnthropic(tools) @@ -453,6 +455,9 @@ export async function parseAnthropicCompletion( return { shouldContinue: true, tokenUsage } } + if (finalMessage.stop_reason === 'max_tokens') { + throw new OutputTokenLimitError() + } return { shouldContinue: false, tokenUsage } } diff --git a/frontend/src/lib/components/copilot/chat/app/core.ts b/frontend/src/lib/components/copilot/chat/app/core.ts index 5ab271cc6f..634341b966 100644 --- a/frontend/src/lib/components/copilot/chat/app/core.ts +++ b/frontend/src/lib/components/copilot/chat/app/core.ts @@ -18,6 +18,7 @@ import { type AppCodeSelectionElement, type AppDatatableElement } from '../context' +import { appDatatableRole, sdkDatatableCall } from '$lib/components/raw_apps/dataTableRefUtils' // Backend runnable types export type BackendRunnableType = 'script' | 'flow' | 'hubscript' | 'inline' @@ -921,9 +922,20 @@ export function prepareAppSystemMessage(customPrompt?: string): ChatCompletionSy const policy = aiChatManager.datatableCreationPolicy const datatableName = policy.datatable ?? 'main' const schemaPrefix = policy.schema ? `${policy.schema}.` : '' - // Use wmill.datatable() for 'main' (default), otherwise wmill.datatable('name') - const datatableCall = - datatableName === 'main' ? 'wmill.datatable()' : `wmill.datatable('${datatableName}')` + // A role names the privileges the app's queries run with, so it has to be in the code the + // model writes. + const datatableRole = appDatatableRole(policy.roles, datatableName) + const tsDatatableCall = sdkDatatableCall(datatableName, datatableRole, 'typescript') + const pyDatatableCall = sdkDatatableCall(datatableName, datatableRole, 'python') + const roleEntries = Object.entries(policy.roles ?? {}) + const rolesNote = + roleEntries.length > 0 + ? `\n\nThis app uses these data tables through a role: ${roleEntries + .map(([dt, role]) => `\`${dt}\` as \`${role}\``) + .join( + ', ' + )}. Always pass that role when calling \`wmill.datatable\` on them, as in the examples. The role only reaches what it was granted, so a query on a table it lacks privileges on fails with \`permission denied\`.` + : '' let content = `You are a helpful assistant that creates and edits apps on the Windmill platform. Apps are defined as a collection of files that contains both the frontend and the backend. @@ -1024,7 +1036,7 @@ Backend runnables should only perform **data operations** (SELECT, INSERT, UPDAT import * as wmill from 'windmill-client'; export async function main(user_id: string) { - const sql = ${datatableCall}; + const sql = ${tsDatatableCall}; const user = await sql\`SELECT * FROM ${schemaPrefix}users WHERE id = \${user_id}\`.fetchOne(); return user; } @@ -1035,12 +1047,12 @@ export async function main(user_id: string) { import wmill def main(user_id: str): - db = ${datatableCall} + db = ${pyDatatableCall} user = db.query('SELECT * FROM ${schemaPrefix}users WHERE id = $1', user_id).fetch_one() return user \`\`\` -Use these examples for normal datatable access. +Use these examples for normal datatable access.${rolesNote} ### Schema Modifications (DDL) - Use exec_datatable_sql tool ONLY diff --git a/frontend/src/lib/components/copilot/chat/chatLoop.test.ts b/frontend/src/lib/components/copilot/chat/chatLoop.test.ts index 251c2ff858..144617779f 100644 --- a/frontend/src/lib/components/copilot/chat/chatLoop.test.ts +++ b/frontend/src/lib/components/copilot/chat/chatLoop.test.ts @@ -8,6 +8,7 @@ import { type ChatLoopConfig } from './chatLoop' import type { ReasoningProviderModel } from '../reasoningRegistry' +import { OutputTokenLimitError } from './outputTokenLimit' const mocks = vi.hoisted(() => ({ getCompletion: vi.fn(), @@ -540,6 +541,23 @@ describe('runChatLoop lastIterationUsage', () => { }) }) +describe('runChatLoop output token limit', () => { + beforeEach(() => { + vi.resetAllMocks() + mocks.resolveRequestReasoning.mockReturnValue(undefined) + }) + + it('fails a truncated Responses iteration instead of replaying it on the Completions API', async () => { + mocks.getOpenAIResponsesCompletion.mockResolvedValue({}) + mocks.parseOpenAIResponsesCompletion.mockRejectedValue(new OutputTokenLimitError()) + + await expect( + runChatLoop(createConfig({ workspace: `workspace-${randomUUID()}` })) + ).rejects.toBeInstanceOf(OutputTokenLimitError) + expect(mocks.getCompletion).not.toHaveBeenCalled() + }) +}) + // Builders for the message shapes the chat loop accumulates. const assistant = (content: string): ChatCompletionMessageParam => ({ role: 'assistant', content }) const assistantTools = (...ids: string[]): ChatCompletionMessageParam => ({ diff --git a/frontend/src/lib/components/copilot/chat/chatLoop.ts b/frontend/src/lib/components/copilot/chat/chatLoop.ts index 110e4f1154..5091d76af5 100644 --- a/frontend/src/lib/components/copilot/chat/chatLoop.ts +++ b/frontend/src/lib/components/copilot/chat/chatLoop.ts @@ -20,6 +20,7 @@ import { parseOpenAIResponsesCompletion } from './openai-responses' import type { Tool, ToolCallbacks } from './shared' +import { OutputTokenLimitError } from './outputTokenLimit' import { sanitizeToolCallArguments } from './toolCallArguments' import { addChatTokenUsage, emptyChatTokenUsage, type ChatTokenUsage } from './tokenUsage' @@ -501,6 +502,11 @@ export async function runChatLoop(config: ChatLoopConfig): Promise void dequeueMessage: () => void setComposerStaged: (key: string, editingIndex: number | null, bytes: number) => void + /** Whether this composer holds anything unsent (text, pastes, attachments, reads in flight). */ + setComposerHasDraft: (key: string, hasDraft: boolean) => void clearComposerStaged: (key: string) => void attachmentBytesExcluding: (selfKey: string) => number diff --git a/frontend/src/lib/components/copilot/chat/createdResourceActions.svelte.ts b/frontend/src/lib/components/copilot/chat/createdResourceActions.svelte.ts index bd0f3b3d0e..c76e892cdf 100644 --- a/frontend/src/lib/components/copilot/chat/createdResourceActions.svelte.ts +++ b/frontend/src/lib/components/copilot/chat/createdResourceActions.svelte.ts @@ -5,6 +5,10 @@ type MaybePromise = T | Promise type ToolDisplayActionHandler = (action: ToolDisplayAction) => MaybePromise const toolDisplayActionHandlers = $state>({}) +// Every registration per type, latest last: a page that takes over a type from the layout +// (the sessions page opens items in its panel rather than in drawers) hands it back on +// unmount instead of leaving the type unhandled. +const registrations: Record = {} function formatUnknownError(error: unknown): string { if (error instanceof Error) { @@ -17,11 +21,16 @@ export function registerToolDisplayActionHandler( type: ToolDisplayAction['type'], handler: ToolDisplayActionHandler ): () => void { + const stack = (registrations[type] ??= []) + stack.push(handler) toolDisplayActionHandlers[type] = handler return () => { - if (toolDisplayActionHandlers[type] === handler) { - delete toolDisplayActionHandlers[type] - } + const at = stack.lastIndexOf(handler) + if (at < 0) return + stack.splice(at, 1) + const current = stack[stack.length - 1] + if (current) toolDisplayActionHandlers[type] = current + else delete toolDisplayActionHandlers[type] } } diff --git a/frontend/src/lib/components/copilot/chat/datatableTools.ts b/frontend/src/lib/components/copilot/chat/datatableTools.ts index 7232500898..bb7976d396 100644 --- a/frontend/src/lib/components/copilot/chat/datatableTools.ts +++ b/frontend/src/lib/components/copilot/chat/datatableTools.ts @@ -2,6 +2,7 @@ import { z } from 'zod' import { WorkspaceService, type CompletedJob } from '$lib/gen' import type { DataTableTables } from '$lib/gen/types.gen' import { runScript } from '$lib/components/jobs/utils' +import { datatableReference } from '$lib/components/dbTypes' import { createToolDef, executeTestRun, @@ -15,9 +16,9 @@ import { * * Datatables are workspace-level managed PostgreSQL databases. The backend * endpoints used here (`list_datatable_tables`, `get_datatable_table_schema`) - * and SQL execution (`datatable://`) are gated only by workspace - * membership, so these tools need no app context and operate directly on the - * workspace. This is the unrestricted counterpart to the app-mode datatable + * and SQL execution (`datatable://`) need no app context: the server + * decides what the caller reaches, as the datatable role they name or its + * default. This is the unrestricted counterpart to the app-mode datatable * tools in `app/core.ts`, which additionally filter by the app's whitelist. */ @@ -31,9 +32,19 @@ const memo = (factory: () => T): (() => T) => { // ============= Pure workspace-scoped operations ============= -/** List all datatables configured in the workspace, with their schema/table names. */ -export async function listDatatables(workspace: string): Promise { - return await WorkspaceService.listDataTableTables({ workspace }) +/** List the datatables configured in the workspace, with their schema/table names: all of them as + * their default role, or only `datatableName`, as `role` when one is given. */ +export async function listDatatables( + workspace: string, + datatableName?: string, + role?: string +): Promise { + if (datatableName === undefined) return await WorkspaceService.listDataTableTables({ workspace }) + return await WorkspaceService.listDataTableTables({ + workspace, + datatableName, + ...(role !== undefined && { roleFor: datatableName, role }) + }) } /** Get the columns (column_name -> compact_type) of one datatable table. */ @@ -41,13 +52,15 @@ export async function getDatatableColumns( workspace: string, datatableName: string, schemaName: string, - tableName: string + tableName: string, + role?: string ): Promise> { const schema = await WorkspaceService.getDataTableTableSchema({ workspace, datatableName, schemaName, - tableName + tableName, + role }) return schema.columns } @@ -81,7 +94,26 @@ const NO_DATATABLES_CONFIGURED_MESSAGE = // ============= Tool definitions ============= -const getListDatatablesSchema = memo(() => z.object({})) +// The same rule the server applies to `-- role `; a name it would refuse fails here instead. +const getRoleSchema = memo(() => + z + .string() + .regex(/^[A-Za-z0-9_-]{1,63}$/) + .optional() + .describe( + "The datatable role to connect as, when the code you are working on uses one (an app's `data.roles` entry, or the `role` it passes to wmill.datatable). Omit for the datatable's default role." + ) +) + +const getListDatatablesSchema = memo(() => + z.object({ + datatable_name: z + .string() + .optional() + .describe('List only this datatable. Required with `role`.'), + role: getRoleSchema() + }) +) const getListDatatablesToolDef = memo(() => createToolDef( getListDatatablesSchema(), @@ -94,7 +126,8 @@ const getGetDatatableTableSchemaSchema = memo(() => z.object({ datatable_name: z.string().describe('The datatable name to inspect, e.g. "main".'), schema_name: z.string().describe('The schema name, e.g. "public".'), - table_name: z.string().describe('The table name to inspect.') + table_name: z.string().describe('The table name to inspect.'), + role: getRoleSchema() }) ) const getGetDatatableTableSchemaToolDef = memo(() => @@ -117,6 +150,7 @@ const getExecDatatableSqlSchema = memo(() => .describe( 'The SQL query to execute. Supports SELECT, INSERT, UPDATE, DELETE, CREATE TABLE, ALTER TABLE, DROP TABLE, etc. For SELECT queries, results are returned as an array of objects. A newly created table will appear in list_datatables automatically.' ), + role: getRoleSchema(), background: z .boolean() .optional() @@ -217,10 +251,18 @@ export function getDatatableTools(): Tool<{}>[] { { def: getListDatatablesToolDef(), planModeSafe: true, - fn: async ({ workspace, toolId, toolCallbacks }) => { + fn: async ({ args, workspace, toolId, toolCallbacks }) => { toolCallbacks.setToolStatus(toolId, { content: 'Listing datatables...' }) try { - const metadata = await listDatatables(workspace) + const parsedArgs = getListDatatablesSchema().parse(args ?? {}) + if (parsedArgs.role !== undefined && parsedArgs.datatable_name === undefined) { + throw new Error('`role` needs `datatable_name`, the datatable it is a role of') + } + const metadata = await listDatatables( + workspace, + parsedArgs.datatable_name, + parsedArgs.role + ) if (metadata.length === 0) { toolCallbacks.setToolStatus(toolId, { content: 'No datatables configured — set one up in workspace settings' @@ -236,7 +278,18 @@ export function getDatatableTools(): Tool<{}>[] { toolCallbacks.setToolStatus(toolId, { content: `Listed ${metadata.length} datatable(s) with ${totalTables} table(s)` }) - return JSON.stringify(metadata, null, 2) + // Only what the model acts on: the roles it may pass, not the creation privileges + // the manager's UI gates on. + return JSON.stringify( + metadata.map((d) => ({ + datatable_name: d.datatable_name, + schemas: d.schemas, + ...(d.error && { error: d.error }), + ...(d.permissioned && { usable_roles: d.usable_roles, default_role: d.default_role }) + })), + null, + 2 + ) } catch (e) { const errorMsg = `Error listing datatables: ${e instanceof Error ? e.message : String(e)}` toolCallbacks.setToolStatus(toolId, { content: errorMsg, error: errorMsg }) @@ -257,7 +310,8 @@ export function getDatatableTools(): Tool<{}>[] { workspace, parsedArgs.datatable_name, parsedArgs.schema_name, - parsedArgs.table_name + parsedArgs.table_name, + parsedArgs.role ) toolCallbacks.setToolStatus(toolId, { content: `Retrieved schema for ${parsedArgs.schema_name}.${parsedArgs.table_name}` @@ -300,7 +354,7 @@ export function getDatatableTools(): Tool<{}>[] { requestBody: { language: 'postgresql', content: parsedArgs.sql, - args: { database: `datatable://${name}` } + args: { database: datatableReference(name, parsedArgs.role) } } }), workspace, diff --git a/frontend/src/lib/components/copilot/chat/global/apiCatalogTools.test.ts b/frontend/src/lib/components/copilot/chat/global/apiCatalogTools.test.ts deleted file mode 100644 index 06109708de..0000000000 --- a/frontend/src/lib/components/copilot/chat/global/apiCatalogTools.test.ts +++ /dev/null @@ -1,350 +0,0 @@ -import { beforeEach, describe, expect, it, vi } from 'vitest' - -const { listMcpToolsMock } = vi.hoisted(() => ({ - listMcpToolsMock: vi.fn() -})) - -vi.mock('../shared', () => ({ - createToolDef: (_schema: unknown, name: string, description: string) => ({ - type: 'function', - function: { name, description, parameters: {} } - }) -})) - -vi.mock('$lib/gen', () => ({ - McpService: { - listMcpTools: listMcpToolsMock - } -})) - -import { apiCatalogTools, clearApiCatalogCache } from './apiCatalogTools' - -const CATALOG = [ - { - name: 'listWorkers', - description: 'List workers', - instructions: 'List all workers with their ping status', - path: '/workers/list', - method: 'GET', - query_params_schema: { - type: 'object', - properties: { page: { type: 'integer' }, per_page: { type: 'integer' } } - } - }, - { - name: 'listDataMetrics', - description: 'List declared measures and dimensions', - instructions: '', - path: '/w/{workspace}/data_metrics/list', - method: 'GET' - }, - { - name: 'listQueue', - description: 'List queued jobs', - instructions: 'List the jobs waiting in the queue', - path: '/w/{workspace}/jobs/queue/list', - method: 'GET', - query_params_schema: { - type: 'object', - properties: { running: { type: 'boolean' }, per_page: { type: 'integer' } } - } - }, - { - name: 'getJobUpdates', - description: 'Get job updates', - instructions: '', - path: '/w/{workspace}/jobs_u/getupdate/{id}', - method: 'GET', - path_params_schema: { - type: 'object', - properties: { workspace: { type: 'string' }, id: { type: 'string' } }, - required: ['workspace', 'id'] - } - }, - { - name: 'getJob', - description: 'Get job details', - instructions: '', - path: '/w/{workspace}/jobs_u/get/{id}', - method: 'GET', - path_params_schema: { - type: 'object', - properties: { workspace: { type: 'string' }, id: { type: 'string' } }, - required: ['workspace', 'id'] - } - }, - { - name: 'deleteSchedule', - description: 'Delete a schedule', - instructions: '', - path: '/w/{workspace}/schedules/delete/{path}', - method: 'DELETE', - path_params_schema: { - type: 'object', - properties: { workspace: { type: 'string' }, path: { type: 'string' } }, - required: ['workspace', 'path'] - } - }, - { - name: 'createFlow', - description: 'Create a flow', - instructions: '', - path: '/w/{workspace}/flows/create', - method: 'POST' - }, - { - name: 'getVariable', - description: 'Get variable', - instructions: '', - path: '/w/{workspace}/variables/get/{path}', - method: 'GET' - }, - { - name: 'getScriptByPath', - description: 'Get script by path', - instructions: '', - path: '/w/{workspace}/scripts/get/p/{path}', - method: 'GET' - }, - { - name: 'deleteScriptByHash', - description: 'Delete a script by hash', - instructions: '', - path: '/w/{workspace}/scripts/delete/h/{hash}', - method: 'POST', - path_params_schema: { - type: 'object', - properties: { workspace: { type: 'string' }, hash: { type: 'string' } }, - required: ['workspace', 'hash'] - } - }, - { - name: 'runScriptByPath', - description: 'Run script by path', - instructions: 'Trigger a run of a deployed script', - path: '/w/{workspace}/jobs/run/p/{path}', - method: 'POST', - path_params_schema: { - type: 'object', - properties: { workspace: { type: 'string' }, path: { type: 'string' } }, - required: ['workspace', 'path'] - }, - body_schema: { type: 'object', additionalProperties: true } - }, - { - name: 'cancelQueuedJob', - description: 'Cancel a queued job', - instructions: '', - path: '/w/{workspace}/jobs_u/queue/cancel/{id}', - method: 'POST', - path_params_schema: { - type: 'object', - properties: { workspace: { type: 'string' }, id: { type: 'string' } }, - required: ['workspace', 'id'] - }, - body_schema: { type: 'object', additionalProperties: true } - }, - { - name: 'runFlowByPath', - description: 'Run flow by path', - instructions: 'Trigger a run of a deployed flow', - path: '/w/{workspace}/jobs/run/f/{path}', - method: 'POST', - path_params_schema: { - type: 'object', - properties: { workspace: { type: 'string' }, path: { type: 'string' } }, - required: ['workspace', 'path'] - }, - body_schema: { - type: 'object', - properties: { args: { type: 'object' } } - } - } -] - -function createToolCallbacks() { - return { - setToolStatus: vi.fn(), - removeToolStatus: vi.fn() - } as any -} - -function getTool(name: string) { - const tool = apiCatalogTools.find((entry) => entry.def.function.name === name) - if (!tool) throw new Error(`${name} tool not found`) - return tool -} - -async function run(name: string, args: Record) { - const raw = await getTool(name).fn({ - args, - workspace: 'test-ws', - helpers: {}, - toolCallbacks: createToolCallbacks(), - toolId: 'tool-1' - }) - return JSON.parse(raw) -} - -beforeEach(() => { - vi.clearAllMocks() - clearApiCatalogCache() - listMcpToolsMock.mockResolvedValue(CATALOG) - vi.unstubAllGlobals() -}) - -describe('search_api_endpoints', () => { - it('matches on name/path tokens, plural-insensitively, and excludes covered endpoints', async () => { - // Singular "job" matches the plural "jobs" path segment. - const result = await run('search_api_endpoints', { query: 'list queued job' }) - expect(result.matches[0].name).toBe('listQueue') - expect(result.matches[0].endpoint).toBe('GET /w/{workspace}/jobs/queue/list') - expect(result.matches[0].params).toEqual(['running', 'per_page']) - expect(result.matches[0].instructions).toContain('waiting in the queue') - - const flows = await run('search_api_endpoints', { query: 'create flow' }) - expect(flows.matches.map((m: any) => m.name)).not.toContain('createFlow') - expect(flows.covered_by_dedicated_tools).toContain('createFlow → use write_flow') - }) - - it('returns endpoint categories when nothing matches', async () => { - const result = await run('search_api_endpoints', { query: 'kubernetes' }) - expect(result.matches).toEqual([]) - expect(result.hint).toContain('jobs_u') - // Categories are built from the uncovered endpoints only, so a covered one - // must not be advertised as somewhere to retry. - expect(result.hint).not.toContain('workers') - expect(result.hint).not.toContain('data_metrics') - }) -}) - -describe('call_api_get', () => { - it('rejects covered, unknown, and non-GET endpoints with a pointer', async () => { - const covered = await run('call_api_get', { name: 'createFlow' }) - expect(covered.error).toContain('write_flow') - - const unknown = await run('call_api_get', { name: 'nope' }) - expect(unknown.error).toContain('search_api_endpoints') - - const mutating = await run('call_api_get', { name: 'cancelQueuedJob' }) - expect(mutating.error).toContain('call_api_endpoint') - - const job = await run('call_api_get', { name: 'getJob' }) - expect(job.error).toContain('get_run') - - const deleting = await run('call_api_endpoint', { name: 'deleteSchedule' }) - expect(deleting.error).toContain('delete_workspace_item') - - const byHash = await run('call_api_endpoint', { name: 'deleteScriptByHash' }) - expect(byHash.error).toContain('delete_workspace_item') - const search = await run('search_api_endpoints', { query: 'delete script' }) - expect(search.matches.map((m: any) => m.name)).not.toContain('deleteScriptByHash') - }) - - // Left reachable, these endpoints are the way around the argument form: they run the - // deployed runnable on the model's arguments, unstripped and unshown. - it('refuses a deployed run, pointing at run_script and run_flow', async () => { - for (const [name, tool, query] of [ - ['runScriptByPath', 'run_script', 'run deployed script'], - ['runFlowByPath', 'run_flow', 'run deployed flow'] - ]) { - const called = await run('call_api_endpoint', { name }) - expect(called.error).toContain(tool) - expect(called.success).toBe(false) - - // And it is gone from search, so the model is redirected before it ever calls. - const search = await run('search_api_endpoints', { query }) - expect(search.matches.map((m: any) => m.name)).not.toContain(name) - expect(search.covered_by_dedicated_tools?.join(' ')).toContain(tool) - } - }) - - it('refuses draft-blind item reads and lists, pointing at the draft-aware tools', async () => { - for (const name of ['getScriptByPath', 'getResource', 'getSchedule']) { - const result = await run('call_api_get', { name }) - expect(result.success).toBe(false) - expect(result.error).toContain('read_workspace_item') - } - for (const name of ['listScripts', 'listFlows', 'listResource', 'listSchedules']) { - const result = await run('call_api_get', { name }) - expect(result.success).toBe(false) - expect(result.error).toContain('list_workspace_items') - } - - const search = await run('search_api_endpoints', { query: 'get script' }) - expect(search.matches.map((m: any) => m.name)).not.toContain('getScriptByPath') - }) - - it('refuses the worker and data-metric reads, pointing at their dedicated tools', async () => { - for (const [name, tool, query] of [ - ['listWorkers', 'list_workers', 'workers'], - ['listDataMetrics', 'list_data_metrics', 'data metrics'] - ]) { - const result = await run('call_api_get', { name }) - expect(result.success).toBe(false) - expect(result.error).toContain(tool) - - const search = await run('search_api_endpoints', { query }) - expect(search.matches.map((m: any) => m.name)).not.toContain(name) - expect(search.covered_by_dedicated_tools?.join(' ')).toContain(tool) - } - }) - - it('refuses variable reads so variable values never reach the model', async () => { - const result = await run('call_api_get', { name: 'getVariable' }) - expect(result.success).toBe(false) - expect(result.error).toContain('never readable') - - const search = await run('search_api_endpoints', { query: 'variable' }) - expect(search.matches.map((m: any) => m.name)).not.toContain('getVariable') - }) - - it('returns the endpoint schema when a required path param is missing', async () => { - const result = await run('call_api_get', { name: 'getJobUpdates' }) - expect(result.success).toBe(false) - expect(result.error).toContain('id') - expect(result.schema.path_params_schema.required).toContain('id') - }) - - it('substitutes path params and sends the rest as query params', async () => { - const fetchMock = vi.fn().mockResolvedValue({ - ok: true, - headers: new Headers({ 'content-type': 'application/json' }), - json: async () => [{ id: 'job-1' }] - }) - vi.stubGlobal('fetch', fetchMock) - const result = await run('call_api_get', { name: 'listQueue', params: { running: true } }) - expect(fetchMock).toHaveBeenCalledWith('/api/w/test-ws/jobs/queue/list?running=true', { - method: 'GET' - }) - expect(result).toEqual({ success: true, data: [{ id: 'job-1' }] }) - }) -}) - -describe('call_api_endpoint', () => { - it('requires confirmation and executes mutating endpoints with a body', async () => { - expect(getTool('call_api_endpoint').requiresConfirmation).toBe(true) - const fetchMock = vi.fn().mockResolvedValue({ - ok: true, - headers: new Headers({ 'content-type': 'text/plain' }), - text: async () => 'job-id-1' - }) - vi.stubGlobal('fetch', fetchMock) - const result = await run('call_api_endpoint', { - name: 'cancelQueuedJob', - params: { id: 'job/1' }, - body: { args: { n: 1 } } - }) - expect(fetchMock).toHaveBeenCalledWith('/api/w/test-ws/jobs_u/queue/cancel/job%2F1', { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ args: { n: 1 } }) - }) - expect(result).toEqual({ success: true, data: 'job-id-1' }) - }) - - it('redirects GET endpoints to call_api_get', async () => { - const result = await run('call_api_endpoint', { name: 'listQueue' }) - expect(result.error).toContain('call_api_get') - }) -}) diff --git a/frontend/src/lib/components/copilot/chat/global/apiCatalogTools.ts b/frontend/src/lib/components/copilot/chat/global/apiCatalogTools.ts deleted file mode 100644 index 42c68c6ebf..0000000000 --- a/frontend/src/lib/components/copilot/chat/global/apiCatalogTools.ts +++ /dev/null @@ -1,391 +0,0 @@ -import { z } from 'zod' -import { McpService, type EndpointTool } from '$lib/gen' -import { createToolDef, type Tool } from '../shared' - -/** - * Generic access to the backend's MCP endpoint catalog (the endpoints marked - * `x-mcp-tool` in openapi.yaml) as three small static tools — search, GET - * call, mutating call — instead of one registered tool per endpoint. Search - * results carry parameter names and usage instructions; full parameter schemas - * enter the model's context only when a call fails validation. This keeps the - * per-iteration tool-schema cost constant. - */ - -// Endpoints whose job a dedicated global tool already does, plus the variable -// read endpoints. Hidden from search and refused at call time: the authoring -// and delete ones would bypass the draft lifecycle (conflict detection, -// explicit deploy, draft cleanup on delete), the variable reads would expose -// variable values to the model (getVariable even decrypts secrets by default), -// and the rest are exact duplicates that would fragment behavior across two -// code paths. -const COVERED_ENDPOINTS: Record = { - getVariable: 'read_workspace_item (variable values are never readable in chat)', - listVariable: 'list_workspace_items (variable values are never readable in chat)', - // The item read/list endpoints return deployed state only, blind to the user's - // drafts; read_workspace_item / list_workspace_items merge drafts, and for - // flows return the compact JSON that patch_flow_json matches against. - getScriptByPath: - 'read_workspace_item (reads your draft when one exists; pass version: "deployed" for the deployed state)', - getFlowByPath: - 'read_workspace_item (reads your draft when one exists; pass version: "deployed" for the deployed state)', - getResource: - 'read_workspace_item (reads your draft when one exists; pass version: "deployed" for the deployed state)', - getSchedule: - 'read_workspace_item (reads your draft when one exists; pass version: "deployed" for the deployed state)', - listScripts: 'list_workspace_items (it includes your drafts)', - listFlows: 'list_workspace_items (it includes your drafts)', - listResource: 'list_workspace_items (it includes your drafts)', - listSchedules: 'list_workspace_items (it includes your drafts)', - runScriptByPath: 'run_script (it shows the user an argument form to confirm)', - runFlowByPath: 'run_flow (it shows the user an argument form to confirm)', - deleteScriptByPath: 'delete_workspace_item', - deleteScriptByHash: 'delete_workspace_item', - deleteFlowByPath: 'delete_workspace_item', - deleteSchedule: 'delete_workspace_item', - deleteVariable: 'delete_workspace_item', - deleteResource: 'delete_workspace_item', - createScript: 'write_script', - createFlow: 'write_flow', - updateFlow: 'patch_flow_json or write_flow', - createApp: 'init_app and the app draft tools', - updateApp: 'write_app_file / write_app_runnable', - createVariable: 'write_variable', - updateVariable: 'write_variable', - createResource: 'write_resource', - updateResource: 'write_resource', - createSchedule: 'write_schedule', - updateSchedule: 'write_schedule', - searchDocs: 'search_docs', - readDocsPage: 'read_docs_page', - listJobs: 'list_runs', - listWorkers: 'list_workers', - listDataMetrics: 'list_data_metrics', - getJob: 'get_run', - getJobLogs: 'get_run', - runScriptPreviewAndWaitResult: 'test_run_script' -} - -const MAX_SEARCH_RESULTS = 10 -const MAX_DESCRIPTION_CHARS = 200 -const MAX_INSTRUCTIONS_CHARS = 400 -const MAX_RESULT_CHARS = 20_000 - -let catalogCache: { workspace: string; endpoints: EndpointTool[] } | undefined - -async function loadCatalog(workspace: string): Promise { - if (catalogCache?.workspace !== workspace) { - const endpoints = await McpService.listMcpTools({ workspace }) - catalogCache = { workspace, endpoints } - } - return catalogCache.endpoints -} - -export function clearApiCatalogCache() { - catalogCache = undefined -} - -function tokenize(text: string): string[] { - return text - .replace(/([a-z0-9])([A-Z])/g, '$1 $2') - .toLowerCase() - .split(/[^a-z0-9]+/) - .filter((t) => t.length > 1) -} - -// Cheap plural-insensitive comparison so "workers" matches "worker" and vice versa. -function tokenMatches(token: string, queryToken: string): boolean { - const strip = (t: string) => (t.length > 3 && t.endsWith('s') ? t.slice(0, -1) : t) - return strip(token) === strip(queryToken) -} - -function pathSegments(path: string): string[] { - return path.split('/').filter((seg) => seg && seg !== 'w' && !seg.startsWith('{')) -} - -// Name and path tokens identify the operation; description tokens only support it. -function scoreEndpoint(endpoint: EndpointTool, queryTokens: string[]): number { - const nameTokens = [...tokenize(endpoint.name), ...pathSegments(endpoint.path).flatMap(tokenize)] - const descTokens = tokenize(`${endpoint.description} ${endpoint.instructions}`) - let score = 0 - for (const qt of queryTokens) { - if (nameTokens.some((t) => tokenMatches(t, qt))) score += 3 - else if (descTokens.some((t) => tokenMatches(t, qt))) score += 1 - } - return score -} - -function schemaPropertyNames(schema: unknown): string[] { - const properties = (schema as { properties?: Record } | null | undefined) - ?.properties - return properties ? Object.keys(properties).filter((k) => k !== 'workspace') : [] -} - -function truncate(text: string, max: number): string { - return text.length > max ? text.slice(0, max) + '…' : text -} - -function summarizeEndpoint(endpoint: EndpointTool) { - const params = [ - ...schemaPropertyNames(endpoint.path_params_schema), - ...schemaPropertyNames(endpoint.query_params_schema) - ] - const bodyParams = schemaPropertyNames(endpoint.body_schema) - const instructions = endpoint.instructions.trim() - return { - name: endpoint.name, - endpoint: `${endpoint.method.toUpperCase()} ${endpoint.path}`, - description: truncate(endpoint.description, MAX_DESCRIPTION_CHARS), - ...(instructions ? { instructions: truncate(instructions, MAX_INSTRUCTIONS_CHARS) } : {}), - ...(params.length > 0 ? { params } : {}), - ...(bodyParams.length > 0 ? { body_params: bodyParams } : {}) - } -} - -function endpointSchemaHelp(endpoint: EndpointTool) { - return { - name: endpoint.name, - endpoint: `${endpoint.method.toUpperCase()} ${endpoint.path}`, - path_params_schema: endpoint.path_params_schema ?? undefined, - query_params_schema: endpoint.query_params_schema ?? undefined, - body_schema: endpoint.body_schema ?? undefined - } -} - -async function resolveEndpoint( - workspace: string, - name: string -): Promise<{ endpoint: EndpointTool } | { error: string }> { - const covered = COVERED_ENDPOINTS[name] - if (covered) { - return { error: `"${name}" is covered by the dedicated ${covered} tool — use it instead.` } - } - const catalog = await loadCatalog(workspace) - const endpoint = catalog.find((e) => e.name === name) - if (!endpoint) { - return { - error: `Unknown endpoint "${name}". Use search_api_endpoints to find the endpoint name.` - } - } - return { endpoint } -} - -async function executeEndpoint( - endpoint: EndpointTool, - workspace: string, - params: Record, - body?: unknown -): Promise { - let url = `/api${endpoint.path.replace('{workspace}', encodeURIComponent(workspace))}` - const queryParams = new URLSearchParams() - for (const [key, value] of Object.entries(params)) { - if (value === undefined || value === null) continue - if (url.includes(`{${key}}`)) { - url = url.replace(`{${key}}`, encodeURIComponent(String(value))) - } else { - queryParams.append(key, String(value)) - } - } - const unresolved = [...url.matchAll(/\{([^}]+)\}/g)].map((m) => m[1]) - if (unresolved.length > 0) { - return JSON.stringify({ - success: false, - error: `Missing required path parameter(s): ${unresolved.join(', ')}`, - schema: endpointSchemaHelp(endpoint) - }) - } - const search = queryParams.toString() - if (search) url += `?${search}` - - const method = endpoint.method.toUpperCase() - const fetchOptions: RequestInit = { method } - if (body !== undefined && method !== 'GET') { - fetchOptions.headers = { 'Content-Type': 'application/json' } - fetchOptions.body = JSON.stringify(body) - } - - const response = await fetch(url, fetchOptions) - const raw = response.headers.get('content-type')?.includes('application/json') - ? await response.json() - : await response.text() - if (!response.ok) { - return JSON.stringify({ - success: false, - status: response.status, - error: typeof raw === 'string' ? raw : JSON.stringify(raw), - // 4xx usually means wrong arguments — echo the schema so the model can - // self-correct in the next call without a separate schema-fetch tool. - ...(response.status >= 400 && response.status < 500 - ? { schema: endpointSchemaHelp(endpoint) } - : {}) - }) - } - const result = JSON.stringify({ success: true, data: raw }) - if (result.length <= MAX_RESULT_CHARS) return result - return JSON.stringify({ - success: true, - truncated: true, - data: (typeof raw === 'string' ? raw : JSON.stringify(raw)).slice(0, MAX_RESULT_CHARS), - note: `Result truncated to ${MAX_RESULT_CHARS} characters. Use filter or pagination parameters to narrow it.` - }) -} - -const searchApiEndpointsSchema = z.object({ - query: z - .string() - .describe( - 'Keywords matched against endpoint names, paths, and descriptions (e.g. "queue", "run flow", "audit log"). Jobs are called "runs" in the UI.' - ) -}) - -const callApiGetSchema = z.object({ - name: z.string().describe('Endpoint name as returned by search_api_endpoints'), - params: z - .record(z.string(), z.any()) - .optional() - .describe( - 'Path and query parameter values, keyed by parameter name. The workspace parameter is filled automatically.' - ) -}) - -const callApiEndpointSchema = z.object({ - name: z.string().describe('Endpoint name as returned by search_api_endpoints'), - params: z - .record(z.string(), z.any()) - .optional() - .describe( - 'Path and query parameter values, keyed by parameter name. The workspace parameter is filled automatically.' - ), - body: z - .record(z.string(), z.any()) - .optional() - .describe('JSON request body, when the endpoint takes one') -}) - -export const apiCatalogTools: Tool<{}>[] = [ - { - def: createToolDef( - searchApiEndpointsSchema, - 'search_api_endpoints', - 'Search the Windmill REST API endpoint catalog for operations no dedicated tool covers (queue state, job details, running deployed items, deletions, ...). Returns endpoint names to pass to call_api_get or call_api_endpoint.' - ), - planModeSafe: true, - fn: async ({ args, workspace, toolId, toolCallbacks }) => { - const parsed = searchApiEndpointsSchema.parse(args) - toolCallbacks.setToolStatus(toolId, { content: 'Searching API endpoints...' }) - const catalog = await loadCatalog(workspace) - const queryTokens = tokenize(parsed.query) - const available = catalog.filter((e) => !COVERED_ENDPOINTS[e.name]) - const scored = available - .map((endpoint) => ({ endpoint, score: scoreEndpoint(endpoint, queryTokens) })) - .filter((s) => s.score > 0) - .sort((a, b) => b.score - a.score || a.endpoint.name.localeCompare(b.endpoint.name)) - const coveredHits = catalog - .filter((e) => COVERED_ENDPOINTS[e.name] && scoreEndpoint(e, queryTokens) > 0) - .map((e) => `${e.name} → use ${COVERED_ENDPOINTS[e.name]}`) - - if (scored.length === 0) { - const categories = [...new Set(available.map((e) => pathSegments(e.path)[0]))].sort() - const result = JSON.stringify( - { - matches: [], - hint: `No endpoint matched. Available endpoint categories: ${categories.join(', ')}. Retry with different keywords, or use a dedicated tool if one covers the need.`, - ...(coveredHits.length > 0 ? { covered_by_dedicated_tools: coveredHits } : {}) - }, - null, - 2 - ) - toolCallbacks.setToolStatus(toolId, { content: 'No matching API endpoint', result }) - return result - } - const top = scored.slice(0, MAX_SEARCH_RESULTS) - const result = JSON.stringify( - { - matches: top.map((s) => summarizeEndpoint(s.endpoint)), - ...(scored.length > top.length - ? { - note: `${scored.length - top.length} more match(es) — refine the query to see them.` - } - : {}), - ...(coveredHits.length > 0 ? { covered_by_dedicated_tools: coveredHits } : {}) - }, - null, - 2 - ) - toolCallbacks.setToolStatus(toolId, { - content: `Found ${top.length} API endpoint(s) for "${parsed.query}"`, - result - }) - return result - } - }, - { - def: createToolDef( - callApiGetSchema, - 'call_api_get', - 'Call a read-only GET endpoint from the API catalog by name. Use search_api_endpoints first to find the endpoint name; a failed call returns the parameter schema.' - ), - // Readonly rests on the method check below plus the catalog only exposing - // side-effect-free GETs; never mark a mutating GET as an `x-mcp-tool`. - planModeSafe: true, - showDetails: true, - fn: async ({ args, workspace, toolId, toolCallbacks }) => { - const parsed = callApiGetSchema.parse(args) - const resolved = await resolveEndpoint(workspace, parsed.name) - if ('error' in resolved) { - toolCallbacks.setToolStatus(toolId, { content: resolved.error, error: resolved.error }) - return JSON.stringify({ success: false, error: resolved.error }) - } - if (resolved.endpoint.method.toUpperCase() !== 'GET') { - const error = `"${parsed.name}" is a ${resolved.endpoint.method.toUpperCase()} endpoint — use call_api_endpoint for mutating calls.` - toolCallbacks.setToolStatus(toolId, { content: error, error }) - return JSON.stringify({ success: false, error }) - } - toolCallbacks.setToolStatus(toolId, { content: `Calling ${parsed.name}...` }) - const result = await executeEndpoint(resolved.endpoint, workspace, parsed.params ?? {}) - const ok = JSON.parse(result).success === true - toolCallbacks.setToolStatus(toolId, { - content: ok ? `Called ${parsed.name}` : `Call to ${parsed.name} failed`, - result, - ...(ok ? {} : { error: `Call to ${parsed.name} failed` }) - }) - return result - } - }, - { - def: createToolDef( - callApiEndpointSchema, - 'call_api_endpoint', - 'Call a mutating (POST/PUT/PATCH/DELETE) endpoint from the API catalog by name; the user is asked to confirm. Use search_api_endpoints first to find the endpoint name; a failed call returns the parameter schema.' - ), - requiresConfirmation: true, - confirmationMessage: (args) => `Call API endpoint ${args?.name ?? ''}`, - showDetails: true, - fn: async ({ args, workspace, toolId, toolCallbacks }) => { - const parsed = callApiEndpointSchema.parse(args) - const resolved = await resolveEndpoint(workspace, parsed.name) - if ('error' in resolved) { - toolCallbacks.setToolStatus(toolId, { content: resolved.error, error: resolved.error }) - return JSON.stringify({ success: false, error: resolved.error }) - } - if (resolved.endpoint.method.toUpperCase() === 'GET') { - const error = `"${parsed.name}" is a GET endpoint — use call_api_get (no confirmation needed).` - toolCallbacks.setToolStatus(toolId, { content: error, error }) - return JSON.stringify({ success: false, error }) - } - toolCallbacks.setToolStatus(toolId, { content: `Calling ${parsed.name}...` }) - const result = await executeEndpoint( - resolved.endpoint, - workspace, - parsed.params ?? {}, - parsed.body - ) - const ok = JSON.parse(result).success === true - toolCallbacks.setToolStatus(toolId, { - content: ok ? `Called ${parsed.name}` : `Call to ${parsed.name} failed`, - result, - ...(ok ? {} : { error: `Call to ${parsed.name} failed` }) - }) - return result - } - } -] diff --git a/frontend/src/lib/components/copilot/chat/global/core.test.ts b/frontend/src/lib/components/copilot/chat/global/core.test.ts index c35c5518dd..22777015fd 100644 --- a/frontend/src/lib/components/copilot/chat/global/core.test.ts +++ b/frontend/src/lib/components/copilot/chat/global/core.test.ts @@ -260,6 +260,11 @@ vi.mock('$lib/gen', async () => { WorkerService: wrapService(actual.WorkerService, { listWorkers: vi.fn(async () => []) }), + // Guests off by default, as a fresh workspace has them. + WorkspaceService: wrapService(actual.WorkspaceService, { + getGuestUsage: vi.fn(async () => ({ available: false, instance_enabled: false })), + getPublicSettings: vi.fn(async () => ({ guest_access_enabled: false })) + }), FolderService: wrapService(actual.FolderService, { createFolder: vi.fn(async () => 'created') }), @@ -405,7 +410,8 @@ import { ScriptService, UserService, VariableService, - WorkerService + WorkerService, + WorkspaceService } from '$lib/gen' import { devopsRole, superadmin, userStore, usersWorkspaceStore } from '$lib/stores' import { processSecretArgs } from '$lib/components/secretArgUtils' @@ -3591,6 +3597,26 @@ describe('global AI tools', () => { }) }) + // The draft carries the deployed policy from the fork, so nothing is fetched to answer. + it('reports exposure for an app that has a draft over it', async () => { + seedBackendDraft( + 'raw_app', + 'f/apps/drafted', + { + files: { '/src/App.tsx': 'x' }, + runnables: {}, + policy: { execution_mode: 'anonymous' } + } as any, + { workspace: WORKSPACE } + ) + + const read = await callGlobalTool('read_workspace_item', { + type: 'app', + path: 'f/apps/drafted' + }) + expect(JSON.parse(read)).toMatchObject({ isDraft: true, executionMode: 'anonymous' }) + }) + it('summarizes local raw app drafts in read_workspace_item', async () => { seedBackendDraft( 'raw_app', @@ -3706,6 +3732,90 @@ describe('global AI tools', () => { expect(getBackendDraft('raw_app', 'f/apps/report', { workspace: WORKSPACE })).toBeUndefined() }) + // A low-code app has a grid, not files and runnables, so every app tool here would + // otherwise report it as empty rather than say it is the wrong kind of app. + it('reports a low-code app without its contents, and refuses to act on it', async () => { + const lowCode = { + path: 'f/apps/legacy', + summary: 'legacy app', + raw_app: false, + value: { grid: [{ id: 'a', data: { type: 'buttoncomponent' } }] } + } as any + // ...Once per call: a persistent implementation would outlive this test and + // disarm the factory's "mock not configured" guard for the rest of the file. + for (let i = 0; i < 3; i++) vi.mocked(AppService.getAppByPath).mockResolvedValueOnce(lowCode) + + // The read answers "this app is drag-and-drop" rather than throwing — but it must not + // summarize a grid as a file/runnable list, which reads as an empty app. + const read = JSON.parse( + await callGlobalTool('read_workspace_item', { type: 'app', path: 'f/apps/legacy' }) + ) + expect(read).toMatchObject({ type: 'app', path: 'f/apps/legacy', rawApp: false }) + expect(read).not.toHaveProperty('value') + + // The tools that would convert it to files and runnables still refuse: a staged draft + // would answer every later read in place of the app itself. + for (const [tool, args] of [ + ['read_app_file', { path: 'f/apps/legacy', file_path: '/index.tsx' }], + ['write_app_file', { path: 'f/apps/legacy', file_path: '/App.tsx', content: 'x' }] + ] as [string, any][]) { + const raw = await callGlobalTool(tool, args).catch((e) => String(e)) + expect(raw).toContain('low-code app') + } + expect(getBackendDraft('raw_app', 'f/apps/legacy', { workspace: WORKSPACE })).toBeUndefined() + }) + + // This read is the only way to ask who may open a deployed app. Both kinds answer: a + // drag-and-drop app can be anonymous too, and no other tool here can inspect one. + it('reports who may open an app, whichever kind it is', async () => { + const readMode = async (app: any) => { + vi.mocked(AppService.getAppByPath).mockResolvedValueOnce(app) + const read = await callGlobalTool('read_workspace_item', { type: 'app', path: app.path }) + return JSON.parse(read).executionMode + } + const guestApp = { + path: 'f/apps/code', + summary: 'Code app', + raw_app: true, + value: { files: {}, runnables: {} }, + policy: { execution_mode: 'guest' } + } + + vi.mocked(WorkspaceService.getGuestUsage).mockResolvedValueOnce({ + available: true, + instance_enabled: true + } as any) + vi.mocked(WorkspaceService.getPublicSettings).mockResolvedValueOnce({ + guest_access_enabled: true + } as any) + expect(await readMode(guestApp)).toBe('guest') + + expect( + await readMode({ + path: 'f/apps/builder', + summary: 'Builder app', + raw_app: false, + value: { grid: [] }, + policy: { execution_mode: 'anonymous' } + }) + ).toBe('anonymous') + + // The same app, with each switch crossed in turn: either one alone admits nobody, so + // reporting the mode bare would name an exposure the server refuses. Both off needs + // no case of its own — whichever half of the check were dropped, one of these two + // still catches it. + vi.mocked(WorkspaceService.getPublicSettings).mockResolvedValueOnce({ + guest_access_enabled: true + } as any) + expect(await readMode(guestApp)).toContain('inert') + + vi.mocked(WorkspaceService.getGuestUsage).mockResolvedValueOnce({ + available: true, + instance_enabled: true + } as any) + expect(await readMode(guestApp)).toContain('inert') + }) + it('reads raw app files without creating a draft', async () => { vi.mocked(AppService.getAppByPath).mockResolvedValueOnce({ path: 'f/apps/report', @@ -4362,6 +4472,69 @@ describe('global AI tools', () => { expect(getBackendDraft('raw_app', 'f/apps/report', { workspace: WORKSPACE })).toBeUndefined() }) + // Deploying is what makes an app's runnables reachable, so it is the one moment the + // exposure is both true and known. + it('discloses who can open an app after a deploy', async () => { + const deployApp = async (path: string, policy: Record) => { + vi.mocked(AppService.existsApp).mockResolvedValueOnce(true) + vi.mocked(AppService.getAppByPath).mockResolvedValueOnce({} as any) + seedBackendDraft( + 'raw_app', + path, + { + summary: 'App', + files: { '/index.tsx': 'x' }, + runnables: {}, + data: { tables: [] }, + policy + }, + { workspace: WORKSPACE } + ) + return JSON.parse(await callGlobalTool('deploy_workspace_item', { type: 'app', path })) + } + + const open = await deployApp('f/apps/open', { + execution_mode: 'anonymous', + on_behalf_of: 'u/alice' + }) + expect(open.success).toBe(true) + expect(open.message).toContain('anyone with the URL, without logging in') + // The server decides what a deployed app runs as — it overwrites on_behalf_of for + // anyone outside the deployers group — so the note must not name an identity. + expect(open.message).not.toContain('u/alice') + + // Guest is a real widening only where the deployment, the instance and the + // workspace all admit guests; stored below that, it is inert. + vi.mocked(WorkspaceService.getGuestUsage).mockResolvedValueOnce({ + available: true, + instance_enabled: true + } as any) + vi.mocked(WorkspaceService.getPublicSettings).mockResolvedValueOnce({ + guest_access_enabled: true + } as any) + const guestOn = await deployApp('f/apps/guest', { execution_mode: 'guest' }) + expect(guestOn.success).toBe(true) + expect(guestOn.message).toContain('identity provider authenticates') + + // Each switch crossed in turn, because either one alone admits nobody and the note + // would then announce an exposure that does not exist. Both off needs no case of its + // own: whichever half of the check were dropped, one of these two still catches it. + vi.mocked(WorkspaceService.getPublicSettings).mockResolvedValueOnce({ + guest_access_enabled: true + } as any) + const instanceOff = await deployApp('f/apps/guest_inst_off', { execution_mode: 'guest' }) + expect(instanceOff.success).toBe(true) + expect(instanceOff.message).not.toContain('identity provider') + + vi.mocked(WorkspaceService.getGuestUsage).mockResolvedValueOnce({ + available: true, + instance_enabled: true + } as any) + const workspaceOff = await deployApp('f/apps/guest_ws_off', { execution_mode: 'guest' }) + expect(workspaceOff.success).toBe(true) + expect(workspaceOff.message).not.toContain('identity provider') + }) + it('forwards preserve_on_behalf_of when the deployed policy carries an on_behalf_of', async () => { // Without the flag the backend resets the policy's on_behalf_of to the // deploying user; this chat path has no on-behalf-of selector, so it must @@ -6956,8 +7129,7 @@ describe('session-only preview tools gating', () => { expect(names).not.toContain('list_app_runs') expect(names).not.toContain('search_dom') expect(names).not.toContain('read_dom') - // Not withheld: without it the side panel's only route to a deployed run is the raw - // endpoint, which confirms an opaque request body instead of the arguments. + // Not withheld: without it the side panel has no route to a deployed run at all. expect(names).toContain('run_script') // other tools are still present expect(names).toContain('write_script') diff --git a/frontend/src/lib/components/copilot/chat/global/core.ts b/frontend/src/lib/components/copilot/chat/global/core.ts index 9ead934dff..7973a90464 100644 --- a/frontend/src/lib/components/copilot/chat/global/core.ts +++ b/frontend/src/lib/components/copilot/chat/global/core.ts @@ -19,7 +19,8 @@ import { SqsTriggerService, VariableService, WebsocketTriggerService, - WorkerService + WorkerService, + WorkspaceService } from '$lib/gen' import { createTwoFilesPatch } from 'diff' import { deepEqual } from 'fast-equals' @@ -255,7 +256,6 @@ const VARIABLE_MASKED_NOTE = 'Note: variable values are never shown in chat — the diff marks whether the value changed without revealing it.\n\n' const SECRET_UNCOMPARABLE_NOTE = 'Note: this is a SECRET variable — its value is never shown and cannot be compared, so it may ALSO have changed beyond what this diff shows.\n\n' -import { apiCatalogTools } from './apiCatalogTools' const ITEM_TYPES = [ 'script', @@ -310,8 +310,8 @@ export type GlobalActivePreviewContext = { * location: a tab can host a legacy app whose hash is app state, and a filter value * can be free text the user typed. Build it with `previewLocationContext`. */ location: string - /** The row whose drawer is open on that page. The list pages drop the anchor when - * their drawer closes, so its absence means no row is open. */ + /** The item open in its editor: a list page row whose drawer is open, or the item a + * session tab edits. Its absence means no item is open. */ open?: string } @@ -1336,7 +1336,7 @@ const buildGlobalSystemPrompt = ( // right now: the system prompt is the cached prefix, so a line appearing and // disappearing between turns costs more cache than the tool call it saves. const activePreviewRule = previewTools - ? '\n- If the user message includes an ACTIVE PREVIEW section, that is the page the side panel is showing — resolve "this page", "here" and "it" against it, and against `open` (the row the page is anchored at, whose drawer the user opened) when there is one. It already tells you what get_preview_status would, so do not call that tool to learn what is on screen; call it only to check the panel\'s *other* tabs.' + ? '\n- If the user message includes an ACTIVE PREVIEW section, that is the page the side panel is showing — resolve "this page", "here" and "it" against it, and against `open` (the item the user has open in its editor) when there is one. It already tells you what get_preview_status would, so do not call that tool to learn what is on screen; call it only to check the panel\'s *other* tabs.' : '' const pipelineBullet = `- A "data pipeline" is NOT a flow: it is a DAG of independent scripts in one folder, wired by storage assets (DuckLake/data tables/S3) and triggers via top-of-file \`pipeline\` / \`on \` annotation comments written in each script's comment syntax (\`--\` for SQL, \`#\` for Python/Bash, \`//\` for TS — a \`//\` line in a SQL node is a syntax error). When the user asks for a data pipeline (or to ingest/transform/materialize data across steps), call get_instructions with subject "pipeline" and build annotated script drafts — do not build a flow.${pipelineAlphaNote}` // Hosting and edition come from the hostname and a store the app populates at init, so @@ -1387,13 +1387,12 @@ ${pipelineBullet} - Use list_runs to find recent runs (optionally filtered by path, creator, label, or status), then get_run with a returned id to see what that run was called with, what it returned and what it logged — without starting a new test run. - get_run also covers what a flow run did per step — statuses and results across the whole execution tree, subflow steps and loop iterations included — and works while the flow is still running. Pass step to read one step's result in full (capped at 12k chars). - Use open_page to show a workspace page with filters applied — Runs, Schedules, Variables, Resources, Assets, Audit logs, or Workspace settings on a specific tab (e.g. "open the failed runs of f/foo/bar", "open the schedule for X", "open the git sync settings"). Carry over every filter the user described — Runs takes the page's whole filter set (time window, path, user, folder, label, tag, worker, trigger kind, args/result, ...), so don't drop a criterion just because it wasn't in the request's main clause. Only the pages listed for this user in the tool are available; don't offer pages that aren't listed. Don't use it as a substitute for list_runs when you just need the data yourself. -- Whenever you ask the user to perform a manual step in the UI — fill in a resource's credentials, set a secret variable's value, adjust a schedule or setting — call open_page in the same message, targeted at that item (pass open with its path to land in its edit drawer, or the page's filters otherwise). Never just describe where to click. +- Whenever you ask the user to perform a manual step in the UI — fill in a resource's credentials, set a secret variable's value, adjust a schedule or setting — call open_page in the same message, targeted at that item (pass open with its path to land in its editor, or the page's filters otherwise). Never just describe where to click. - When the user is happy with the changes and wants to review or deploy them, use open_page with page "compare" — it opens the Compare & Deploy review page.${ previewTools ? ' By default it preselects the items this chat modified; pass items (":" entries) to control the selection' : ' Pass items (":" entries naming the items you changed) so the review is scoped to them — omitting items preselects every pending change in the workspace' }, or mode ("draft" or "fork") to force which comparison is shown. Prefer offering this review page over calling deploy_workspace_item directly when several items changed. -- For a Windmill operation no other tool covers (queue state, a run's args, ...), use search_api_endpoints to find a REST endpoint, then call_api_get for reads or call_api_endpoint for mutations (the user is asked to confirm those). Always prefer a dedicated tool when one exists; endpoints for authoring or deleting scripts, flows, apps, schedules, resources, or variables are not available through the API catalog tools — use the draft tools and delete_workspace_item instead. - Default to test_run_script, test_run_flow, or test_run_step for any run request, an existing script included; they prefer drafts and need no deployment. Use run_script or run_flow only when the user names the deployed version ("the deployed X", "in production", "for real") — a bare "run X" is not that. For those two, read the item with read_workspace_item version: "deployed" first so the arguments match the deployed schema. test_run_script, test_run_flow, test_run_step, run_script and run_flow all show the user an argument form prefilled with what you sent, so fill in every argument you can infer rather than asking for it in chat. test_run_step's form is the step's own inputs, not the flow's. - When a required decision is ambiguous, use askUserQuestion with two to ten clear proposed answer strings instead of guessing. The user can also type a custom answer when none of the proposed answers fit. Set multiSelect: true only when the answers can genuinely co-apply and the user may pick several (not mutually exclusive). - When the user asks you to remember a lasting preference, always/never do something, or change/stop a behavior going forward, call update_user_instructions to persist it. It edits only the USER INSTRUCTIONS block (not WORKSPACE INSTRUCTIONS). Keep each instruction concise; do not use it for one-off requests scoped to the current task. @@ -1429,8 +1428,9 @@ Flows: - Use patch_flow_json for structural flow edits and write_flow for full flow rewrites. Raw apps: -- The app tools below only work on raw (code) apps. \`rawApp\` says which: false is a drag-and-drop app, which you can list and read but not edit or deploy. Check it before offering to change an app. +- The app tools below only work on raw (code) apps. \`rawApp\` says which: false is a drag-and-drop app: you can list it and read its metadata, but not read its contents, edit it or deploy it. Check it before offering to change an app. - read_workspace_item returns app metadata only. Use read_app_file for file and inline runnable contents. +- A draft app is reachable by nobody; deploying is what exposes its backend runnables. deploy_workspace_item says so when the deploy widens who may open the app: anonymous means anyone with the URL, without logging in; guest means anyone the instance's identity provider authenticates, member of this workspace or not. Relay that in plain words and carry on. This is disclosure, not a gate: do not stop and ask for permission, and do not refuse the deploy. You cannot change who may open an app from chat; it is set on the app's deploy settings. - Use write_app_file, patch_app_file, and delete_app_file for frontend files. - Use write_app_runnable and delete_app_runnable for backend runnables. - Use init_app only after confirming framework, path, and summary with the user. @@ -1440,7 +1440,12 @@ Data Tables: - Datatables are workspace-scoped managed PostgreSQL databases, shared across the workspace (not owned by any single app). They must be configured by the user in their workspace settings (Workspace settings → Data Tables); they cannot be created via SQL. - Use list_datatables to discover the available datatables and their tables. Reuse an existing table rather than creating a duplicate. If list_datatables reports none, this is a blocking prerequisite — tell the user to set up a datatable in their workspace settings and stop; do not assume a "main" datatable exists or call exec_datatable_sql. - Use get_datatable_table_schema only when you need a table's column names/types; list_datatables is enough for table-list or availability summaries. -- Use exec_datatable_sql to explore data, run queries, mutate rows, or change schema (CREATE/ALTER/DROP). Creating a table is a normal CREATE TABLE statement — it appears in list_datatables afterward, with no registration step. +- Use exec_datatable_sql to explore data, run queries, mutate rows, or change schema (CREATE/ALTER/DROP). Creating a table is a normal CREATE TABLE statement — it appears in list_datatables afterward, with no registration step.${ + isCloudHosted() + ? '' + : ` +- A raw app may use a datatable through a role (\`data.roles\` in its raw_app.yaml). When working on such an app, pass that role to the datatable tools, and to wmill.datatable in its runnables, so you see and change only what the app itself can.` + } - When writing runnable code (inline app runnables, scripts, flow modules) that reads or writes datatable data at runtime, it accesses a datatable via wmill.datatable(). Default to TypeScript (bun) unless the user asked for another language. Call get_instructions with subject "datatable" and language "bun" for the TypeScript SQL SDK reference (or language "python3" for Python) — it returns only that language so you get just what you need.${ skills.length > 0 ? ` @@ -1536,6 +1541,7 @@ function serializeWorkspaceItemForRead(item: WorkspaceItem): unknown { summary: item.summary, value: summarizeAppValue(item.value as AppDraftValue), rawApp: item.rawApp, + executionMode: item.executionMode, isDraft: item.isDraft } } @@ -1832,13 +1838,28 @@ function getInlineRunnableContent( return { content: runnable.inlineScript?.content ?? '', runnable } } +// appSourceToDraftValue drops a low-code app's `grid`, so converting one here would stage a +// code-app draft at its path: a false picture of the app that every later read answers from, +// and a deploy the server then refuses. +async function getRawAppByPath(workspace: string, path: string): Promise { + const app = await AppService.getAppByPath({ workspace, path }) + // Only an explicit false: the draft-only branch of get_app carries no version, and so + // no `raw_app`, and must not read as low-code. + if (app.raw_app === false) { + throw new Error( + `"${path}" is a low-code app. This chat only edits code-based apps — open it in the app editor instead.` + ) + } + return app +} + async function loadAppValueForRead(path: string, workspace: string): Promise { const draft = await getGlobalDraft(workspace, 'app', path) if (draft && draft.value && typeof draft.value === 'object' && 'files' in draft.value) { return draft.value as AppDraftValue } - const app = await AppService.getAppByPath({ workspace, path }) + const app = await getRawAppByPath(workspace, path) return appSourceToDraftValue(app, app) } @@ -1848,7 +1869,7 @@ async function loadAppDraftValue(path: string, workspace: string): Promise = { } } +/** Whether the server would admit a guest here: the deployment has to support guests at + * all, and the instance and workspace switches both have to be on. `guest` is stored on + * an app even when none of that holds, so the mode alone never settles who can open it. + * `undefined` when a switch read fails — neither proven live nor proven inert. */ +async function guestAccessIsLive(workspace: string): Promise { + const [usage, settings] = await Promise.all([ + WorkspaceService.getGuestUsage({ workspace }).catch(() => undefined), + WorkspaceService.getPublicSettings({ workspace }).catch(() => undefined) + ]) + if (usage === undefined || settings === undefined) { + return undefined + } + return !!(usage.available && usage.instance_enabled && settings.guest_access_enabled) +} + +/** Who may open an app, from the mode stored on it. An app sits in `guest` mode whether + * or not anyone is admitted by it, so reporting the mode bare would say strangers can + * open an app that admits members only. Say it is inert rather than hide it, as the + * app's deploy settings do. */ +async function describeAppExposure( + workspace: string, + mode: string | undefined +): Promise { + if (mode !== 'guest' || (await guestAccessIsLive(workspace)) !== false) { + return mode + } + return 'guest, but inert: the instance or workspace admits no guest, so this app admits members only' +} + async function readWorkspaceItem( type: WorkspaceItemType, path: string, @@ -2044,6 +2094,19 @@ async function readWorkspaceItem( case 'app': { // Returns lightweight metadata only — file/runnable contents come via read_app_file. const app = await AppService.getAppByPath({ workspace, path }) + // A grid is not files and runnables: summarizing one reports an empty app. Name the + // kind instead. + const executionMode = await describeAppExposure(workspace, app.policy?.execution_mode) + if (app.raw_app === false) { + return { + type: 'app', + path: app.path, + summary: app.summary, + rawApp: false, + executionMode, + isDraft: false + } + } const value = appSourceToDraftValue(app) const metadata = summarizeAppValue(value) return { @@ -2052,6 +2115,7 @@ async function readWorkspaceItem( summary: value.summary, value: metadata as unknown as AppDraftValue, rawApp: app.raw_app, + executionMode, isDraft: false } } @@ -2798,7 +2862,7 @@ const openPageFullSchema = z.object({ .string() .optional() .describe( - 'Schedules/Triggers/Variables/Resources: exact item path to open in the edit drawer, e.g. f/foo/my_schedule. Use it whenever the user should act on one specific item (e.g. fill in credentials) so they land directly in its editor.' + 'Schedules/Triggers/Variables/Resources: exact item path to open in its editor, e.g. f/foo/my_schedule — in a session, a preview tab of its own instead of the list page. Use it whenever the user should act on one specific item (e.g. fill in credentials) so they land directly in its editor.' ), summary: z .string() @@ -2927,7 +2991,7 @@ function buildOpenPageDefSchema( } const OPEN_PAGE_DESCRIPTION = - 'Open a Windmill page with filters applied — Runs, Schedules, Variables, Resources, Assets, Audit logs, Folders, Groups, Triggers (by kind), Workspace settings (on a specific tab), or the Compare & Deploy review page. Inside an AI session it opens as a tab in the side-panel preview next to the chat; elsewhere it offers a clickable link. Use after surfacing something the user likely wants to inspect (e.g. "show me the failed runs of X", "open the schedule for Y", "open the git sync settings", "open the kafka triggers"), and ALWAYS when asking the user to perform a manual step themselves (fill in a resource\'s credentials, set a variable\'s value — pass open with the item path so its edit drawer opens directly). Use page "compare" when the user wants to review and deploy pending changes (the items field controls which changes are preselected). This is the only way to show one of these pages in the session preview — open_preview only handles editable items (scripts, flows, raw apps, pipelines). Only pages listed for this user are available; do not offer others.' + 'Open a Windmill page with filters applied — Runs, Schedules, Variables, Resources, Assets, Audit logs, Folders, Groups, Triggers (by kind), Workspace settings (on a specific tab), or the Compare & Deploy review page. Inside an AI session it opens as a tab in the side-panel preview next to the chat; elsewhere it offers a clickable link. Use after surfacing something the user likely wants to inspect (e.g. "show me the failed runs of X", "open the schedule for Y", "open the git sync settings", "open the kafka triggers"), and ALWAYS when asking the user to perform a manual step themselves (fill in a resource\'s credentials, set a variable\'s value — pass open with the item path so its editor opens directly). Use page "compare" when the user wants to review and deploy pending changes (the items field controls which changes are preselected). This is the only way to show one of these pages in the session preview — open_preview only handles editable items (scripts, flows, raw apps, pipelines). Only pages listed for this user are available; do not offer others.' // Non-arg inputs the URL builder needs: the chat's operating workspace (the compare // page cannot fall back to its own store default inside a session preview) and the @@ -3478,7 +3542,17 @@ export const globalTools: Tool<{}>[] = [ toolCallbacks.setToolStatus(toolId, { content: `Read draft ${parsed.type} "${parsed.path}"` }) - return JSON.stringify(serializeWorkspaceItemForRead(draft), null, 2) + // The mode reported is the draft's, which is what deploying it will write. The + // deployed app's own mode is a different question, asked with version: + // "deployed", and must not be fetched here. + const executionMode = + parsed.type === 'app' + ? await describeAppExposure( + workspace, + (draft.value as AppDraftValue)?.policy?.execution_mode + ) + : undefined + return JSON.stringify(serializeWorkspaceItemForRead({ ...draft, executionMode }), null, 2) } toolCallbacks.setToolStatus(toolId, { @@ -4364,10 +4438,7 @@ export const globalTools: Tool<{}>[] = [ // Workspace DuckLake: pipeline storage prerequisite, and declared measures ...getDucklakeTools(), // Read-only tools over files the user attached to the conversation - ...fileTools, - // Search + call access to the backend API endpoint catalog, for operations - // no dedicated tool covers - ...apiCatalogTools + ...fileTools ] // Tools that only make sense inside an AI session (they drive the session's @@ -8008,6 +8079,27 @@ async function deployDraft( `not working until they are.` } + // `policy` is the mode being written, so the exposure is stated from the value in + // hand. What a runnable runs as is the server's to decide, so the note names who + // can reach the app and never an identity. + if (policy.execution_mode === 'anonymous') { + deployNote = + `${deployNote ? `${deployNote} ` : ''}This app is deployed as anonymous: its ` + + `backend runnables are now reachable by anyone with the URL, without logging in. ` + + `Tell the user plainly what is now reachable and by whom.` + } else if (policy.execution_mode === 'guest') { + // Only where the guest door actually opens: below that, silence — a false note + // is worse than none. The standing cap is a live count no read settles, so the + // note says the door is open, not that every newcomer gets in. + if (await guestAccessIsLive(workspace)) { + deployNote = + `${deployNote ? `${deployNote} ` : ''}This app is deployed as guest: anyone ` + + `the instance's identity provider authenticates can now open it and run its ` + + `backend runnables, member of this workspace or not. Tell the user plainly ` + + `what is now reachable and by whom.` + } + } + toolCallbacks.setToolStatus(toolId, { content: `Bundling app "${path}"...` }) diff --git a/frontend/src/lib/components/copilot/chat/global/flowRunTree.ts b/frontend/src/lib/components/copilot/chat/global/flowRunTree.ts index 7b81a70d7c..af203e2c8c 100644 --- a/frontend/src/lib/components/copilot/chat/global/flowRunTree.ts +++ b/frontend/src/lib/components/copilot/chat/global/flowRunTree.ts @@ -400,8 +400,7 @@ function shapeRunResult(job: Job): Record { : { result: cap(stringify(job.result)) } } -/** Why a run ended badly. get_run is the only route to these — the catalog - * refuses getJob. Kept out of summarizeRun, which list_runs pays per job. */ +/** Why a run ended badly. Kept out of summarizeRun, which list_runs pays per job. */ function diagnoseRun(job: Job): Record { return { ...(job.canceled_by ? { canceled_by: job.canceled_by } : {}), diff --git a/frontend/src/lib/components/copilot/chat/global/globalAssembly.test.ts b/frontend/src/lib/components/copilot/chat/global/globalAssembly.test.ts new file mode 100644 index 0000000000..feffda425e --- /dev/null +++ b/frontend/src/lib/components/copilot/chat/global/globalAssembly.test.ts @@ -0,0 +1,63 @@ +import { describe, expect, it, vi } from 'vitest' + +vi.mock('monaco-editor', () => ({ + editor: {}, + languages: {}, + KeyCode: {}, + Uri: { parse: (value: string) => ({ toString: () => value }) }, + MarkerSeverity: { Error: 8, Warning: 4, Info: 2, Hint: 1 } +})) +vi.mock('@codingame/monaco-vscode-standalone-typescript-language-features', () => ({ + getTypeScriptWorker: async () => async () => ({}), + typescriptVersion: 'test' +})) +vi.mock('@codingame/monaco-vscode-languages-service-override', () => ({ default: () => ({}) })) +vi.mock('$lib/components/vscode', () => ({})) + +import { getSessionContextPromptSection, type SessionPromptContext } from './core' +import { getPipelinePromptSection, type PipelineContext } from '../pipeline/core' +import { assembleGlobalSystemMessage, assembleGlobalTools } from './globalAssembly' + +const sessionContext: SessionPromptContext = { workspaceId: 'ws', parentWorkspaceId: 'parent' } +const pipelineContext: PipelineContext = { + folder: 'myfolder', + mode: 'edit', + nodes: [], + assets: [] +} + +describe('assembleGlobalTools', () => { + it('includes the pipeline tools only when a pipeline editor is active', () => { + const names = (opts: Parameters[0]) => + assembleGlobalTools(opts).map((t) => t.def.function.name) + expect(names({ pipelineContext })).toContain('get_pipeline_graph') + expect(names({})).not.toContain('get_pipeline_graph') + }) +}) + +describe('assembleGlobalSystemMessage', () => { + // The hazard this module exists to remove: every caller gets every section, so a + // mid-session rebuild (update_user_instructions) cannot silently drop one. + it('carries both optional sections when both contexts are given', () => { + const content = assembleGlobalSystemMessage(undefined, { + sessionContext, + pipelineContext + }).content as string + expect(content).toContain(getSessionContextPromptSection(sessionContext)) + expect(content).toContain(getPipelinePromptSection(pipelineContext)) + // Order is part of the assembled output, so it is pinned too. + expect(content.indexOf(getSessionContextPromptSection(sessionContext))).toBeLessThan( + content.indexOf(getPipelinePromptSection(pipelineContext)) + ) + }) + + it('omits them when the contexts are absent', () => { + const content = assembleGlobalSystemMessage(undefined, {}).content as string + // Each section's own heading, read from the code rather than retyped: a reword + // cannot quietly make these vacuous, and a heading is the one part every variant + // shares, so a stray section built from any other context still trips them. + const heading = (s: string) => s.trim().split('\n')[0] + expect(content).not.toContain(heading(getSessionContextPromptSection(sessionContext))) + expect(content).not.toContain(heading(getPipelinePromptSection(pipelineContext))) + }) +}) diff --git a/frontend/src/lib/components/copilot/chat/global/globalAssembly.ts b/frontend/src/lib/components/copilot/chat/global/globalAssembly.ts new file mode 100644 index 0000000000..1988f33dd9 --- /dev/null +++ b/frontend/src/lib/components/copilot/chat/global/globalAssembly.ts @@ -0,0 +1,50 @@ +/** + * The whole assembled surface of a GLOBAL-mode chat — prompt sections and tools — + * in one place, so a new section cannot be added to one rebuild path and forgotten + * on another. Plan mode's decoration is not here: it is applied per request, later. + */ +import type { ChatCompletionSystemMessageParam } from 'openai/resources/index.mjs' +import type { Tool } from '../shared' +import { + getSessionContextPromptSection, + globalToolsFor, + prepareGlobalSystemMessage, + type SessionPromptContext +} from './core' +import { createMcpTools } from './mcpTools' +import { getPipelinePromptSection, pipelineTools, type PipelineContext } from '../pipeline/core' +import { filterToolsForAgentAccess } from '../agentAccessPolicy' + +// Derived, not retyped: an option added to prepareGlobalSystemMessage is reachable +// here at once. Hand-listing the four would compile fine while leaving the new one +// unreachable from this path, and only the eval harness — which calls the builder +// directly — would get it. +export type GlobalAssemblyOpts = NonNullable[1]> & { + sessionContext?: SessionPromptContext + pipelineContext?: PipelineContext +} + +export function assembleGlobalSystemMessage( + instructions: Parameters[0], + opts: GlobalAssemblyOpts +): ChatCompletionSystemMessageParam { + const systemMessage = prepareGlobalSystemMessage(instructions, opts) + if (opts.sessionContext) { + systemMessage.content += getSessionContextPromptSection(opts.sessionContext) + } + if (opts.pipelineContext) { + systemMessage.content += getPipelinePromptSection(opts.pipelineContext) + } + return systemMessage +} + +export function assembleGlobalTools(opts: GlobalAssemblyOpts): Tool[] { + return [ + ...filterToolsForAgentAccess( + globalToolsFor({ sessionPreview: opts.previewTools ?? false }), + opts.accessPolicy + ), + ...(opts.pipelineContext ? pipelineTools : []), + ...createMcpTools(opts.mcpServers ?? []) + ] +} diff --git a/frontend/src/lib/components/copilot/chat/global/workspaceItems.ts b/frontend/src/lib/components/copilot/chat/global/workspaceItems.ts index 61b2773ef8..3d922a3b72 100644 --- a/frontend/src/lib/components/copilot/chat/global/workspaceItems.ts +++ b/frontend/src/lib/components/copilot/chat/global/workspaceItems.ts @@ -143,6 +143,14 @@ export type WorkspaceItem = { * editor. The two are edited by disjoint tool sets, so the distinction has to * reach the model before it picks one. */ rawApp?: boolean + /** Apps only. Who may open the app — read from a draft, who may open it once + * that draft is deployed: `anonymous` is anyone with the URL and no login, + * `guest` anyone the identity provider authenticates, the rest a workspace + * member. A `guest` app whose instance or workspace admits + * no guest says so here, since the mode is stored either way. Reported per + * app, not in listings — a listing carrying it would make absence read as + * proof of no exposure. */ + executionMode?: string isDraft: boolean isLiveDraft?: boolean } diff --git a/frontend/src/lib/components/copilot/chat/mention.test.ts b/frontend/src/lib/components/copilot/chat/mention.test.ts index 8d379ce3b4..ae021bf1de 100644 --- a/frontend/src/lib/components/copilot/chat/mention.test.ts +++ b/frontend/src/lib/components/copilot/chat/mention.test.ts @@ -1,5 +1,12 @@ import { describe, expect, it } from 'vitest' -import { MENTION_RE, mentionTitle, formatMention } from './mention' +import { + MENTION_RE, + mentionTitle, + formatMention, + hasMention, + mentionTitlesInText, + removeMentionFromText +} from './mention' describe('formatMention', () => { it('leaves a simple name bare', () => { @@ -52,3 +59,83 @@ describe('MENTION_RE', () => { expect(mentionTitle(m[0])).toBe(name) }) }) + +describe('removeMentionFromText', () => { + it('removes a bare mention and keeps neighboring words separated', () => { + expect(removeMentionFromText('before @app.ts after', 'app.ts')).toBe('before after') + }) + + it('removes a bracketed mention', () => { + expect(removeMentionFromText('before @[my folder/a b.ts] after', 'my folder/a b.ts')).toBe( + 'before after' + ) + }) + + it('removes an escaped bracketed mention', () => { + expect(removeMentionFromText('before @[notes \\] draft.md] after', 'notes ] draft.md')).toBe( + 'before after' + ) + }) + + it('removes edge mentions without leaving extra whitespace', () => { + expect(removeMentionFromText('@app.ts after', 'app.ts')).toBe('after') + expect(removeMentionFromText('before @app.ts', 'app.ts')).toBe('before') + }) + + it('keeps embedded mention-like text intact', () => { + expect(removeMentionFromText('Contact owner@app.ts about @app.ts', 'app.ts')).toBe( + 'Contact owner@app.ts about' + ) + }) + + it('keeps Unicode-prefixed embedded mention-like text intact', () => { + expect(removeMentionFromText('Contact 用户@app.ts about @app.ts', 'app.ts')).toBe( + 'Contact 用户@app.ts about' + ) + expect(removeMentionFromText('Contact हिंदी@app.ts about @app.ts', 'app.ts')).toBe( + 'Contact हिंदी@app.ts about' + ) + expect(removeMentionFromText('Contact 𐐀@app.ts about @app.ts', 'app.ts')).toBe( + 'Contact 𐐀@app.ts about' + ) + }) +}) + +describe('hasMention', () => { + it('does not treat embedded mention-like text as a selected context mention', () => { + expect(hasMention('Contact owner@app.ts', 'app.ts')).toBe(false) + expect(hasMention('Contact owner@app.ts about @app.ts', 'app.ts')).toBe(true) + }) + + it('does not treat Unicode-prefixed embedded text as a mention', () => { + expect(hasMention('Contact 用户@app.ts', 'app.ts')).toBe(false) + expect(hasMention('Contact हिंदी@app.ts', 'app.ts')).toBe(false) + expect(hasMention('Contact 𐐀@app.ts', 'app.ts')).toBe(false) + }) + + it('treats punctuation next to a mention as a boundary', () => { + expect(hasMention('use @app.ts, then compare', 'app.ts')).toBe(true) + expect(hasMention('open (@app.ts)', 'app.ts')).toBe(true) + }) +}) + +describe('mentionTitlesInText', () => { + it('extracts only standalone mentions for context synchronization', () => { + expect([...mentionTitlesInText('Contact owner@app.ts about @app.ts')]).toEqual(['app.ts']) + expect([...mentionTitlesInText('Contact owner@app.ts')]).toEqual([]) + }) + + it('ignores Unicode-prefixed embedded text for context synchronization', () => { + expect([...mentionTitlesInText('Contact 用户@app.ts')]).toEqual([]) + }) + + it('keeps punctuation-adjacent mentions synchronized', () => { + expect([...mentionTitlesInText('use @app.ts, then compare')]).toEqual(['app.ts']) + expect([...mentionTitlesInText('open (@app.ts)')]).toEqual(['app.ts']) + }) + + it('ignores combining-mark and astral-letter embedded text', () => { + expect([...mentionTitlesInText('Contact हिंदी@app.ts')]).toEqual([]) + expect([...mentionTitlesInText('Contact 𐐀@app.ts')]).toEqual([]) + }) +}) diff --git a/frontend/src/lib/components/copilot/chat/mention.ts b/frontend/src/lib/components/copilot/chat/mention.ts index 65a2b3b7c6..d1d42696b0 100644 --- a/frontend/src/lib/components/copilot/chat/mention.ts +++ b/frontend/src/lib/components/copilot/chat/mention.ts @@ -29,3 +29,59 @@ const BARE_SAFE = /^[\w/.\-]+$/ export function formatMention(name: string): string { return BARE_SAFE.test(name) ? `@${name}` : `@[${name.replace(/[\\\]]/g, '\\$&')}]` } + +function isWordChar(char: string | undefined): boolean { + return char !== undefined && /[\p{L}\p{N}\p{M}_]/u.test(char) +} + +function codePointBefore(text: string, index: number): string | undefined { + if (index <= 0) return undefined + const prev = text.charCodeAt(index - 1) + if (prev >= 0xdc00 && prev <= 0xdfff && index > 1) return text.slice(index - 2, index) + return text[index - 1] +} + +// A mention is owned by this parser when the `@` starts a token. The trailing side is +// intentionally open: non-spacing scripts often continue immediately after a mention, +// while embedded forms like `owner@app.ts` are rejected by the leading boundary. +export function hasMentionLeadingBoundary(text: string, index: number): boolean { + return !isWordChar(codePointBefore(text, index)) +} + +export function isStandaloneMention(text: string, match: RegExpMatchArray): boolean { + if (match.index === undefined) return false + return hasMentionLeadingBoundary(text, match.index) +} + +export function hasMention(text: string, title: string): boolean { + return [...text.matchAll(MENTION_RE)].some( + (m) => mentionTitle(m[0]) === title && isStandaloneMention(text, m) + ) +} + +export function mentionTitlesInText(text: string): Set { + const out = new Set() + for (const m of text.matchAll(MENTION_RE)) { + if (isStandaloneMention(text, m)) out.add(mentionTitle(m[0])) + } + return out +} + +export function removeMentionFromText(text: string, title: string): string { + let out = '' + let last = 0 + for (const m of text.matchAll(MENTION_RE)) { + if (m.index === undefined || mentionTitle(m[0]) !== title) continue + let start = m.index + let end = m.index + m[0].length + if (!isStandaloneMention(text, m)) continue + const hasLead = start > 0 && /\s/.test(text[start - 1]) + const hasTrail = end < text.length && /\s/.test(text[end]) + if (hasLead && !hasTrail) start -= 1 + if (!hasLead && hasTrail) end += 1 + if (hasLead && hasTrail) end += 1 + out += text.slice(last, start) + last = end + } + return out + text.slice(last) +} diff --git a/frontend/src/lib/components/copilot/chat/openai-responses.test.ts b/frontend/src/lib/components/copilot/chat/openai-responses.test.ts index 276084a171..951f8f0c77 100644 --- a/frontend/src/lib/components/copilot/chat/openai-responses.test.ts +++ b/frontend/src/lib/components/copilot/chat/openai-responses.test.ts @@ -3,8 +3,10 @@ import { buildPromptCacheKey, getOpenAIResponsesCompletion, openAIWebSearchDetails, + parseOpenAIResponsesCompletion, toResponsesContent } from './openai-responses' +import { OutputTokenLimitError } from './outputTokenLimit' const mocks = vi.hoisted(() => ({ getProviderAndCompletionConfig: vi.fn(), @@ -153,3 +155,34 @@ describe('openAIWebSearchDetails', () => { ).toEqual({ query: undefined, sources: [{ url: 'https://a.dev' }] }) }) }) + +describe('parseOpenAIResponsesCompletion output token limit', () => { + it('fails a response the stream reports incomplete at max_output_tokens', async () => { + const handlers: Record void> = {} + // The SDK's final snapshot of an incomplete response still reads in_progress; + // only the response.incomplete event carries the reason. + const runner = { + on: (name: string, fn: (event: any) => void) => { + handlers[name] = fn + }, + done: async () => { + handlers['response.incomplete']?.({ + type: 'response.incomplete', + response: { incomplete_details: { reason: 'max_output_tokens' } } + }) + }, + finalResponse: async () => ({ status: 'in_progress', output: [] }) + } + + const parsed = parseOpenAIResponsesCompletion( + runner as any, + { onNewToken: vi.fn(), onMessageEnd: vi.fn(), setToolStatus: vi.fn() } as any, + [], + [], + [], + {} + ) + + await expect(parsed).rejects.toBeInstanceOf(OutputTokenLimitError) + }) +}) diff --git a/frontend/src/lib/components/copilot/chat/openai-responses.ts b/frontend/src/lib/components/copilot/chat/openai-responses.ts index 4118eb492d..0e460f1921 100644 --- a/frontend/src/lib/components/copilot/chat/openai-responses.ts +++ b/frontend/src/lib/components/copilot/chat/openai-responses.ts @@ -21,6 +21,7 @@ import { type ToolCallbacks, type WebSearchSource } from './shared' +import { OutputTokenLimitError } from './outputTokenLimit' import type { ResponseStream } from 'openai/lib/responses/ResponseStream.mjs' import type { AIProviderModel } from '$lib/gen' import { openAIResponsesUsageToChatTokenUsage, type ChatTokenUsage } from './tokenUsage' @@ -271,7 +272,8 @@ export async function getOpenAIResponsesCompletion( messages, stream: true, tools, - forceModelProvider: options?.forceModelProvider + forceModelProvider: options?.forceModelProvider, + reasoningEffort: options?.reasoningEffort }) const { instructions, input } = convertMessagesToResponsesInput(messages) const responsesConfig = applyReasoningToConfig( @@ -330,7 +332,8 @@ export async function* getOpenAIResponsesCompletionStream( messages, stream: true, tools, - forceModelProvider: options?.forceModelProvider + forceModelProvider: options?.forceModelProvider, + reasoningEffort: options?.reasoningEffort }) const { instructions, input } = convertMessagesToResponsesInput(messages) // No prompt cache key here: a rejected key has to be retried without it, and this is @@ -542,6 +545,13 @@ export async function parseOpenAIResponsesCompletion( }) }) + // The stream's final snapshot keeps the in-progress status, so an + // incomplete response is only visible through its own event. + let hitOutputTokenLimit = false + runner.on('response.incomplete', (event) => { + hitOutputTokenLimit = event.response.incomplete_details?.reason === 'max_output_tokens' + }) + // Handle errors runner.on('error', (err: OpenAIError | ResponseErrorEvent) => { currentStreamingTool = undefined @@ -600,6 +610,9 @@ export async function parseOpenAIResponsesCompletion( return { shouldContinue: true, tokenUsage } } + if (hitOutputTokenLimit) { + throw new OutputTokenLimitError() + } return { shouldContinue: false, tokenUsage } } diff --git a/frontend/src/lib/components/copilot/chat/outputTokenLimit.ts b/frontend/src/lib/components/copilot/chat/outputTokenLimit.ts new file mode 100644 index 0000000000..19b74f0c68 --- /dev/null +++ b/frontend/src/lib/components/copilot/chat/outputTokenLimit.ts @@ -0,0 +1,17 @@ +/** The provider ended the response at the request's output token cap. A turn + * cut off there must fail rather than end quietly: thinking counts toward the + * cap, so the stop often lands mid-thought and would read as the model giving + * up. Failing also keeps the turn's output so far for a follow-up. */ +export class OutputTokenLimitError extends Error { + constructor() { + super( + "The response was cut off at the model's output token limit. Ask it to continue, or raise the limit in the workspace AI settings under Model output limits." + ) + this.name = 'OutputTokenLimitError' + } +} + +/** Shown instead on Windmill's free tier, whose server clamps the output below any + * workspace setting: pointing at that setting would send the user to a no-op. */ +export const FREE_TIER_OUTPUT_TOKEN_LIMIT_MESSAGE = + "The response was cut off at the free tier's output token limit. Ask it to continue, or add your own API key in the workspace AI settings for a higher limit." diff --git a/frontend/src/lib/components/copilot/chat/script/core.ts b/frontend/src/lib/components/copilot/chat/script/core.ts index 35e6f5169a..622f182eae 100644 --- a/frontend/src/lib/components/copilot/chat/script/core.ts +++ b/frontend/src/lib/components/copilot/chat/script/core.ts @@ -22,9 +22,10 @@ import { } from '../shared' import { createWorkspaceMutationTools } from '../workspaceTools' import { setupTypeAcquisition, type DepsToGet } from '$lib/ata' -import { getModelContextWindow } from '../../modelConfig' +import { getEffectiveModelContextWindow } from '../../modelConfig' import type { ReviewChangesOpts } from '../monaco-adapter' -import { getCurrentModel } from '$lib/aiStore' +import { copilotInfo, getCurrentModel } from '$lib/aiStore' +import { get } from 'svelte/store' import { getDbSchemas } from '$lib/components/apps/components/display/dbtable/metadata' import { getScriptPrompt, getWorkflowAsCodePrompt } from '$system_prompts' @@ -548,7 +549,11 @@ export async function searchExternalIntegrationResources(args: { query: string } ) const model = getCurrentModel() - const modelContextWindow = getModelContextWindow(model.model) + const modelContextWindow = getEffectiveModelContextWindow( + model.provider, + model.model, + get(copilotInfo).contextWindowPerModel + ) const results: PackageSearchResult[] = await Promise.all( filtered.map(async (r: PackageSearchQuery) => { let documentation = '' diff --git a/frontend/src/lib/components/copilot/chat/shared.test.ts b/frontend/src/lib/components/copilot/chat/shared.test.ts index d1f8706089..b9f01768d0 100644 --- a/frontend/src/lib/components/copilot/chat/shared.test.ts +++ b/frontend/src/lib/components/copilot/chat/shared.test.ts @@ -1551,6 +1551,50 @@ describe('pollJobCompletion detach', () => { }) }) +describe('executeTestRun result note', () => { + // A failed run's card lands on the error, with the logs a tab away: the model must stay + // free to quote them, so only a success says the user already sees the run. + it('tells the model not to repeat a successful result, and not a failed one', async () => { + vi.useFakeTimers() + try { + const { executeTestRun } = await import('./shared') + const { JobService } = await import('$lib/gen') + const getJobUpdates = vi.mocked(JobService.getJobUpdates) + getJobUpdates.mockReset() + getJobUpdates.mockResolvedValue({ completed: true, running: false } as any) + const run = async (success: boolean) => { + vi.mocked(JobService.getJob).mockReset() + vi.mocked(JobService.getJob).mockResolvedValue({ + type: 'CompletedJob', + success, + result: [1, 2], + logs: 'ran' + } as any) + const promise = executeTestRun({ + jobStarter: async () => 'job1', + workspace: 'w', + toolId: 'tool1', + startMessage: 'Starting...', + contextName: 'script', + // The job hooks are what mark the global/sessions chat. + toolCallbacks: { + setToolStatus: vi.fn(), + removeToolStatus: vi.fn(), + onJobStatus: vi.fn(), + onJobStarted: vi.fn() + } as any + }) + await vi.advanceTimersByTimeAsync(1000) + return promise + } + expect(await run(true)).toContain('Do not repeat') + expect(await run(false)).not.toContain('Do not repeat') + } finally { + vi.useRealTimers() + } + }) +}) + describe('deriveChatJobStatus', () => { // CompletedJob is discriminated by the presence of a `success` key; the branch // order deliberately mirrors JobStatusIcon so the badge and scalar never drift. diff --git a/frontend/src/lib/components/copilot/chat/shared.ts b/frontend/src/lib/components/copilot/chat/shared.ts index 1d1fda292f..fa8f268083 100644 --- a/frontend/src/lib/components/copilot/chat/shared.ts +++ b/frontend/src/lib/components/copilot/chat/shared.ts @@ -785,11 +785,11 @@ export function pendingUserActionDetail( // the sessions page) react to mutating tools — refreshing previews — without // the tool layer knowing about the UI. Single slot; the consumer filters by name // and reads the tool args (e.g. the mutated item's `path`) to scope its refresh. -let toolCompletionListener: ((toolName: string, args: any) => void) | undefined +// `workspace` is the one the tool acted on: a path names an item only within it. +export type ToolCompletionListener = (toolName: string, args: any, workspace: string) => void +let toolCompletionListener: ToolCompletionListener | undefined -export function setToolCompletionListener( - fn: ((toolName: string, args: any) => void) | undefined -): void { +export function setToolCompletionListener(fn: ToolCompletionListener | undefined): void { toolCompletionListener = fn } @@ -817,7 +817,7 @@ async function callTool({ ) } const result = await tool.fn({ args, workspace, helpers, toolCallbacks, toolId }) - toolCompletionListener?.(functionName, args) + toolCompletionListener?.(functionName, args, workspace) return result } @@ -1875,6 +1875,15 @@ export async function buildTestRunArgs( return parsedArgs } +// Said with the result rather than in the system prompt: there, models still copied a +// result's table or JSON into the reply, under a card that already renders it. Only for +// a successful run in the generic card completedJobToolStatus fills: a failed run's card +// lands on the error with the logs a tab away, and a formatCompletion card shows its own. +// Never on a background completion: its card can be far up the chat by then, so the +// reply may be the only place the user reads the result. +const RESULT_SHOWN_NOTE = + "The user already sees this run's result in its card. Do not repeat the result or logs in your reply: say what it shows, quoting only the values your conclusion rests on." + // The string handed back to the model when a job is backgrounded. It carries the // job id so the model can pull status/args/result/logs on demand (get_run / list_runs), // and tells it the completion will be reported later (notify-only wake). @@ -2019,7 +2028,12 @@ export async function executeTestRun(config: TestRunConfig): Promise { ...(job.success ? {} : { error: getErrorMessage(job.result) }) }) - const summary = formatResultSummary(job.result, job.logs, job.success) + // detachEnabled marks the global/sessions chat; the in-editor chats read the summary + // alone. The card opening on the result is up to each tool: a run card does, a + // generic one only with `autoCollapseDetails: false` at its registration. + const summary = + (detachEnabled && job.success ? `${RESULT_SHOWN_NOTE}\n` : '') + + formatResultSummary(job.result, job.logs, job.success) // get_run only exists in the global/sessions chat (the same hosts that wire // the job hooks) — don't advertise it to in-editor chats. if (detachEnabled && config.contextName === 'flow' && !job.success) { diff --git a/frontend/src/lib/components/copilot/lib.test.ts b/frontend/src/lib/components/copilot/lib.test.ts index 964bf6e703..8e06e0e2b9 100644 --- a/frontend/src/lib/components/copilot/lib.test.ts +++ b/frontend/src/lib/components/copilot/lib.test.ts @@ -13,7 +13,7 @@ import { import { parseFimCompletionChoice } from './fim' import { getKnownModelContextWindow, - getModelContextWindow, + getModelContextWindowFromTable, modelSupportsVision, requiresMaxCompletionTokens } from './modelConfig' @@ -263,6 +263,7 @@ describe('model context windows', () => { expect(getKnownModelContextWindow('gemini-3-flash')).toBe(1000000) expect(getKnownModelContextWindow('gemini-2.5-flash')).toBe(1000000) expect(getKnownModelContextWindow('deepseek-v4-pro')).toBe(1000000) + expect(getKnownModelContextWindow('deepseek-flash')).toBe(1000000) expect(getKnownModelContextWindow('deepseek-chat')).toBe(1000000) expect(getKnownModelContextWindow('deepseek-reasoner')).toBe(1000000) }) @@ -289,12 +290,12 @@ describe('model context windows', () => { // a version between "qwen3" and "-max" must not claim the 256K entry, and // there is deliberately no qwen family entry (variant windows range 8K–1M) expect(getKnownModelContextWindow('qwen3.8-max')).toBeUndefined() - expect(getModelContextWindow('qwen3.8-max')).toBe(128000) + expect(getModelContextWindowFromTable('qwen3.8-max')).toBe(128000) }) it('returns undefined for unrecognized models, 128K via the defaulting wrapper', () => { expect(getKnownModelContextWindow('some-custom-model')).toBeUndefined() - expect(getModelContextWindow('some-custom-model')).toBe(128000) + expect(getModelContextWindowFromTable('some-custom-model')).toBe(128000) }) }) diff --git a/frontend/src/lib/components/copilot/lib.toolCalls.test.ts b/frontend/src/lib/components/copilot/lib.toolCalls.test.ts index c665cc964e..6966e018d1 100644 --- a/frontend/src/lib/components/copilot/lib.toolCalls.test.ts +++ b/frontend/src/lib/components/copilot/lib.toolCalls.test.ts @@ -285,3 +285,65 @@ describe('parseOpenAICompletion tool call arguments', () => { expect(toolResult.content).toBe('tool ok') }) }) + +describe('output token limit', () => { + it('follows the model through every host that serves it', async () => { + const { getModelMaxTokens } = await import('./lib') + expect(getModelMaxTokens('deepseek', 'deepseek-flash')).toBe(131072) + expect(getModelMaxTokens('azure_foundry', 'DeepSeek-V4-Pro')).toBe(131072) + expect(getModelMaxTokens('openrouter', 'deepseek/deepseek-v4-pro')).toBe(131072) + // gpt-oss reasons whether or not the chat sends an effort. + expect(getModelMaxTokens('groq', 'openai/gpt-oss-120b')).toBe(16000) + expect(getModelMaxTokens('openrouter', 'openai/o3')).toBe(100000) + // A legacy `/thinking` selection is its base model, not the `thinking` row. + expect(getModelMaxTokens('anthropic', 'claude-sonnet-4-5/thinking')).toBe(64000) + }) + + it('gives a model outside the table room to think only when the request asks it to', async () => { + const { getModelMaxTokens } = await import('./lib') + expect(getModelMaxTokens('openrouter', 'x-ai/grok-4.3', 'high')).toBe(32768) + expect(getModelMaxTokens('openrouter', 'x-ai/grok-4.3', 'none')).toBe(8192) + expect(getModelMaxTokens('openrouter', 'x-ai/grok-4.3')).toBe(8192) + // The registry does not know Custom AI models, so an effort says nothing about them. + expect(getModelMaxTokens('customai', 'local-model', 'high')).toBe(8192) + // A self-hosted open-weight model runs at its operator's context length, not the + // vendor's, while a Custom AI proxy in front of a hosted model keeps its row. + expect(getModelMaxTokens('customai', 'deepseek-v4-pro', 'high')).toBe(8192) + expect(getModelMaxTokens('customai', 'gpt-5')).toBe(128000) + }) + + it('sizes the budget from the effort the same request carries', async () => { + const { getCompletion } = await import('./lib') + const create = vi.fn() + + await getCompletion([], new AbortController(), undefined, { + forceModelProvider: { provider: 'mistral', model: 'mistral-medium-latest' }, + openaiClient: { chat: { completions: { create } } } as any, + reasoningEffort: 'high' + }) + + expect(create.mock.calls[0][0]).toMatchObject({ max_tokens: 32768, reasoning_effort: 'high' }) + }) + + it('fails a response cut off at the output token limit instead of ending the turn', async () => { + const { parseOpenAICompletion } = await import('./lib') + const { OutputTokenLimitError } = await import('./chat/outputTokenLimit') + + // Thinking counts toward max_tokens, so the cap can land before any answer. + const parsed = parseOpenAICompletion( + streamOf([ + { choices: [{ delta: { reasoning_content: 'I need to plan the flow so each' } }] }, + { choices: [{ delta: {}, finish_reason: 'length' }] } + ]), + createCallbacks(), + [], + [], + [], + {}, + undefined, + { workspace: 'test' } + ) + + await expect(parsed).rejects.toBeInstanceOf(OutputTokenLimitError) + }) +}) diff --git a/frontend/src/lib/components/copilot/lib.ts b/frontend/src/lib/components/copilot/lib.ts index 19f350a665..b55aa06c92 100644 --- a/frontend/src/lib/components/copilot/lib.ts +++ b/frontend/src/lib/components/copilot/lib.ts @@ -15,11 +15,17 @@ import { get, type Writable } from 'svelte/store' import { OpenAPI, ResourceService, type Script } from '../../gen' import { EDIT_CONFIG, FIX_CONFIG, GEN_CONFIG } from './prompts' import { + getKnownModelMaxOutputTokens, requiresMaxCompletionTokens, usesAnthropicMessagesApi, usesOpenRouterPromptCaching } from './modelConfig' -import { applyReasoningToConfig } from './reasoningRegistry' +import { + applyReasoningToConfig, + requestsReasoning, + stripLegacyThinkingSuffix, + type ReasoningEffort +} from './reasoningRegistry' import { formatResourceTypes } from './utils' import { appendPendingToolImages, @@ -28,6 +34,7 @@ import { type Tool, type ToolCallbacks } from './chat/shared' +import { OutputTokenLimitError } from './chat/outputTokenLimit' import { hasValidToolCallArguments } from './chat/toolCallArguments' import { getNonStreamingOpenAIResponsesCompletion, @@ -116,7 +123,7 @@ export const AI_PROVIDERS: Record = { }, deepseek: { label: 'DeepSeek', - defaultModels: ['deepseek-v4-pro', 'deepseek-v4-flash'] + defaultModels: ['deepseek-v4-pro', 'deepseek-flash'] }, groq: { label: 'Groq', @@ -309,40 +316,31 @@ export async function fetchAvailableModels( return data?.data.map((m) => m.id) ?? [] } -export function getModelMaxTokens(provider: AIProvider, model: string) { - if (model.includes('gpt-5')) { - return 128000 - } else if ( - (provider === 'azure_openai' || provider === 'openai' || provider === 'azure_foundry') && - model.startsWith('o') - ) { - return 100000 - } else if ( - // Raising this further would also raise the worst case of the - // non-streaming completion path, which the Anthropic SDK refuses once - // the request could run past ~10 minutes. - model.includes('claude-sonnet') || - model.includes('claude-haiku') || - model.includes('claude-fable') || - model.includes('claude-mythos') || - // Opus only from 4.5 on. Opus 4.1 and older cap at 32K and fall through - // to the row below. Dots are normalized because OpenRouter writes - // `anthropic/claude-opus-4.5` where Anthropic writes `claude-opus-4-5`. - /claude-opus-(4-(5|6|7|8)|5)(?!\d)/.test(model.replace(/\./g, '-')) || - model.includes('gemini-2.5') || - model.includes('gemini-3') - ) { - return 64000 - } else if (model.includes('gpt-4.1')) { - return 32768 - } else if (model.includes('claude-opus')) { - return 32000 - } else if (model.includes('gpt-4o') || model.includes('codestral')) { - return 16384 - } else if (model.includes('gpt-4-turbo') || model.includes('gpt-3.5')) { - return 4096 - } - return 8192 +// Thinking counts toward max_tokens, so a model outside the table that is asked to +// reason gets more room than the plain fallback. A host rejects a budget above the +// model's cap, so the reasoning fallback must stay within the cap of every model it +// reaches (true of every reasoning model OpenRouter serves outside the table). Custom +// AI never gets it: the registry does not know its models, and the per-model override +// is its way to a larger budget. +const FALLBACK_MAX_TOKENS = 8192 +const REASONING_FALLBACK_MAX_TOKENS = 32768 + +/** + * The output budget a request sends when the workspace sets no override. `reasoningEffort` + * is the effort the request carries, as `resolveRequestReasoning` resolves it. + */ +export function getModelMaxTokens( + provider: AIProvider, + model: string, + reasoningEffort?: ReasoningEffort +): number { + const bareModel = stripLegacyThinkingSuffix(model) + return ( + getKnownModelMaxOutputTokens(provider, bareModel) ?? + (requestsReasoning(provider, bareModel, reasoningEffort) + ? REASONING_FALLBACK_MAX_TOKENS + : FALLBACK_MAX_TOKENS) + ) } // Resolves the completion token cap for a model: the workspace's per-model @@ -350,8 +348,16 @@ export function getModelMaxTokens(provider: AIProvider, model: string) { // Anthropic request paths so both honor the same limit. `cap` bounds the result // (used by short metadata completions, see METADATA_MAX_TOKENS) — a hard ceiling // that wins over both the workspace override and the default. -function resolveMaxTokens(modelProvider: AIProviderModel, cap?: number): number { - const defaultMaxTokens = getModelMaxTokens(modelProvider.provider, modelProvider.model) +function resolveMaxTokens( + modelProvider: AIProviderModel, + cap?: number, + reasoningEffort?: ReasoningEffort +): number { + const defaultMaxTokens = getModelMaxTokens( + modelProvider.provider, + modelProvider.model, + reasoningEffort + ) const modelKey = `${modelProvider.provider}:${modelProvider.model}` let customMaxTokensStore: Record | undefined try { @@ -373,9 +379,10 @@ export const METADATA_MAX_TOKENS = 4096 function getModelSpecificConfig( modelProvider: AIProviderModel, tools?: OpenAI.Chat.Completions.ChatCompletionTool[], - maxTokensCap?: number + maxTokensCap?: number, + reasoningEffort?: ReasoningEffort ) { - const maxTokens = resolveMaxTokens(modelProvider, maxTokensCap) + const maxTokens = resolveMaxTokens(modelProvider, maxTokensCap, reasoningEffort) if ( (modelProvider.provider === 'openai' || modelProvider.provider === 'azure_openai' || @@ -897,7 +904,8 @@ export function getProviderAndCompletionConfig({ tools, forceModelProvider, maxTokensCap, - promptCaching + promptCaching, + reasoningEffort }: { messages: ChatCompletionMessageParam[] stream: K @@ -907,6 +915,9 @@ export function getProviderAndCompletionConfig({ // Opt-in: a cache write costs more than an uncached read, so it only pays off where // the same prefix is sent again. True for the chat loop, false for one-shot calls. promptCaching?: boolean + // The effort the caller will add with `applyReasoningToConfig`. Only sizes the + // output budget here: a request that reasons needs room for the thinking. + reasoningEffort?: ReasoningEffort }): { provider: AIProvider config: K extends true @@ -923,7 +934,7 @@ export function getProviderAndCompletionConfig({ provider: modelProvider.provider, config: { ...providerConfig, - ...getModelSpecificConfig(modelProvider, tools, maxTokensCap), + ...getModelSpecificConfig(modelProvider, tools, maxTokensCap, reasoningEffort), messages: processedMessages, stream } as any @@ -1129,7 +1140,8 @@ export async function getCompletion( stream: true, tools, forceModelProvider: options?.forceModelProvider, - promptCaching: options?.promptCaching + promptCaching: options?.promptCaching, + reasoningEffort: options?.reasoningEffort }) // Use Responses API for OpenAI and Azure OpenAI @@ -1218,6 +1230,7 @@ export async function parseOpenAICompletion( // to the next call, the previous one is demoted to queued. let streamingToolCallId: string | undefined = undefined let malformedFunctionCallError = false + let hitOutputTokenLimit = false let tokenUsage = emptyChatTokenUsage() let answer = '' @@ -1244,6 +1257,9 @@ export async function parseOpenAICompletion( ) { malformedFunctionCallError = true } + if (finishReason === 'length') { + hitOutputTokenLimit = true + } // Mistral nests reasoning inside structured content parts; split them out // so a content delta never leaks "[object Object]" into the answer. @@ -1460,6 +1476,8 @@ export async function parseOpenAICompletion( } messages.push(toolResponse) addedMessages.push(toolResponse) + } else if (hitOutputTokenLimit) { + throw new OutputTokenLimitError() } else { return { shouldContinue: false, tokenUsage } } diff --git a/frontend/src/lib/components/copilot/modelConfig.test.ts b/frontend/src/lib/components/copilot/modelConfig.test.ts index 1788d9c365..4f54eb223a 100644 --- a/frontend/src/lib/components/copilot/modelConfig.test.ts +++ b/frontend/src/lib/components/copilot/modelConfig.test.ts @@ -1,5 +1,27 @@ import { describe, expect, it } from 'vitest' -import { usesAnthropicMessagesApi } from './modelConfig' +import { + getConfiguredModelContextWindow, + getEffectiveModelContextWindow, + usesAnthropicMessagesApi +} from './modelConfig' + +describe('workspace context window overrides', () => { + const overrides = { 'customai:qwen-local': 32_768, 'anthropic:claude-sonnet-5': 200_000 } + + it('wins over the built-in table and the assumed window for its exact provider:model', () => { + expect(getEffectiveModelContextWindow('customai', 'qwen-local', overrides)).toBe(32_768) + expect(getConfiguredModelContextWindow('customai', 'qwen-local', overrides)).toBe(32_768) + expect(getEffectiveModelContextWindow('anthropic', 'claude-sonnet-5', overrides)).toBe(200_000) + }) + + it('does not apply to the same model id served by another provider', () => { + expect(getEffectiveModelContextWindow('openrouter', 'claude-sonnet-5', overrides)).toBe( + 1_000_000 + ) + expect(getConfiguredModelContextWindow('openai', 'qwen-local', overrides)).toBeUndefined() + expect(getEffectiveModelContextWindow('openai', 'qwen-local', overrides)).toBe(128_000) + }) +}) describe('usesAnthropicMessagesApi', () => { it('routes the native Anthropic provider through the Messages API', () => { diff --git a/frontend/src/lib/components/copilot/modelConfig.ts b/frontend/src/lib/components/copilot/modelConfig.ts index cc45a23dac..77d0cf942a 100644 --- a/frontend/src/lib/components/copilot/modelConfig.ts +++ b/frontend/src/lib/components/copilot/modelConfig.ts @@ -66,9 +66,9 @@ export function requiresMaxCompletionTokens(model: string) { // ids (anthropic.claude-sonnet-4-6-...-v1:0, gpt-5.2-2026-01-01) still resolve. // Conservative family fallbacks sit below the explicit entries; models not // listed at all resolve to undefined. Consumers that need a number regardless -// (trim/compaction, the usage indicator) go through getModelContextWindow, -// whose conservative 128K fallback keeps a limit enforced and is surfaced to -// the user as an assumed window. +// (trim/compaction, the usage indicator) go through +// getEffectiveModelContextWindow, whose conservative 128K fallback keeps a +// limit enforced and is surfaced to the user as an assumed window. const MODEL_CONTEXT_WINDOWS: [name: string, contextWindow: number][] = [ // Anthropic — Sonnet/Opus 4.6+ ship a 1M window at standard pricing (GA); // Haiku, older Claude models (3.x, 4.0, 4.1, 4.5) and date-suffixed Claude 4 @@ -96,10 +96,12 @@ const MODEL_CONTEXT_WINDOWS: [name: string, contextWindow: number][] = [ ['gemini-3.1', 1_000_000], ['gemini-3', 1_000_000], ['gemini-2.5', 1_000_000], - // DeepSeek — the V4 family (pro / flash) is 1M. The deepseek-chat / - // deepseek-reasoner aliases were retired 2026-07-24 but can still sit in a - // saved selection, so they keep resolving to the window they had. + // DeepSeek — the V4 family (pro / flash) is 1M; V4.1 Flash is served as + // `deepseek-flash`. The deepseek-chat / deepseek-reasoner aliases were retired + // 2026-07-24 but can still sit in a saved selection, so they keep resolving to + // the window they had. ['deepseek-v4', 1_000_000], + ['deepseek-flash', 1_000_000], ['deepseek-chat', 1_000_000], ['deepseek-reasoner', 1_000_000], ['deepseek', 128_000], @@ -115,6 +117,58 @@ const MODEL_CONTEXT_WINDOWS: [name: string, contextWindow: number][] = [ ['codestral', 32_000] ] +// Output token budgets, matched like the context windows above so one row covers +// every host of a model: native API, Azure AI Foundry, OpenRouter, Together, Bedrock. +// A host rejects any request above its own limit, so each value is one every host of +// the model accepts, not the vendor's maximum. Models not listed fall back in +// `getModelMaxTokens`. +const MODEL_MAX_OUTPUT_TOKENS: [name: string, maxOutputTokens: number][] = [ + // Anthropic — the Anthropic SDK refuses a non-streaming request whose max_tokens + // implies more than ~10 minutes, so these stay below the models' own maximum. + // Opus caps at 32K before 4.5. + ['claude-opus-4.5', 64_000], + ['claude-opus-4.6', 64_000], + ['claude-opus-4.7', 64_000], + ['claude-opus-4.8', 64_000], + ['claude-opus-5', 64_000], + ['claude-opus', 32_000], + ['claude-sonnet', 64_000], + ['claude-haiku', 64_000], + ['claude-fable', 64_000], + ['claude-mythos', 64_000], + // OpenAI + ['gpt-5', 128_000], + ['gpt-4.1', 32_768], + ['gpt-4o', 16_384], + ['gpt-4-turbo', 4_096], + ['gpt-3.5', 4_096], + ['o1', 100_000], + ['o3', 100_000], + ['o4-mini', 100_000], + // Google + ['gemini-3', 64_000], + ['gemini-2.5', 64_000] +] + +// Open-weight models, which a self-hosted server (Custom AI) caps at whatever context +// length its operator set, often below these: vLLM rejects a request whose prompt plus +// max_tokens exceeds it. So these rows only apply to the providers that host them. +const OPEN_WEIGHT_MAX_OUTPUT_TOKENS: [name: string, maxOutputTokens: number][] = [ + // gpt-oss reasons on every request. Bedrock documents 16K, where Groq takes 65536 + ['gpt-oss', 16_000], + // DeepSeek — thinks by default and counts the thinking toward max_tokens. 131072 + // is DeepSeek's own default at the highest effort. R1 is capped by its OpenRouter + // host. + ['deepseek-v4', 131_072], + ['deepseek-flash', 131_072], + ['deepseek-r1', 16_000], + // Mistral + ['codestral', 16_384], + // Models named for thinking reason on every request, whether or not the chat sends + // an effort. Every one OpenRouter lists takes at least this much. + ['thinking', 32_768] +] + // Version separators differ by route to the same model: Anthropic writes // `claude-opus-4-8`, OpenRouter writes `anthropic/claude-opus-4.8`. Collapsing // dots to dashes on both sides keeps one table entry covering every route — @@ -162,9 +216,7 @@ export function buildModelMatchers( // separator is normalized. Only a short segment: a date is digits as well // (`-20251101`) and stays a decoration. strictVariants ? '(?!-\\d{1,3}(?:$|-))' : '', - strictVariants - ? `(?!-(?!(?:v\\d|${DECORATIVE_SUFFIXES.join('|')})$)[a-z])` - : '' + strictVariants ? `(?!-(?!(?:v\\d|${DECORATIVE_SUFFIXES.join('|')})$)[a-z])` : '' ].join('') return [new RegExp(pattern + guards), value] }) @@ -172,8 +224,9 @@ export function buildModelMatchers( /** * The `provider:model` key the workspace AI settings use for their per-model maps - * (`max_tokens_per_model`, `model_pricing`). A bare model id is not enough: the - * same id can be served by more than one provider at different rates. + * (`max_tokens_per_model`, `model_pricing`, `context_window_per_model`). A bare + * model id is not enough: the same id can be served by more than one provider at + * different rates. * * Matched exactly, unlike the fuzzy tables above. Those tables generalize across * every route to one model on purpose; a per-model *setting* must not, or an @@ -196,9 +249,54 @@ export function getKnownModelContextWindow(model: string): number | undefined { return matchModel(MODEL_CONTEXT_WINDOW_MATCHERS, model) } -export function getModelContextWindow(model: string) { - // Trim/compaction logic needs a number; assume a conservative window when unknown. - return getKnownModelContextWindow(model) ?? 128000 +const MODEL_MAX_OUTPUT_TOKEN_MATCHERS = buildModelMatchers(MODEL_MAX_OUTPUT_TOKENS) +const OPEN_WEIGHT_MAX_OUTPUT_TOKEN_MATCHERS = buildModelMatchers(OPEN_WEIGHT_MAX_OUTPUT_TOKENS) + +export function getKnownModelMaxOutputTokens( + provider: AIProvider, + model: string +): number | undefined { + return ( + matchModel(MODEL_MAX_OUTPUT_TOKEN_MATCHERS, model) ?? + (provider === 'customai' ? undefined : matchModel(OPEN_WEIGHT_MAX_OUTPUT_TOKEN_MATCHERS, model)) + ) +} + +/** Trim/compaction logic needs a number; assume a conservative window when unknown. */ +export const ASSUMED_CONTEXT_WINDOW = 128000 + +/** + * What the built-in table states for a model, assuming a conservative window for one + * it does not list. Table only, so it ignores the workspace's `context_window_per_model` + * — deciding how much context to send goes through `getEffectiveModelContextWindow` + * instead. Named for the table because a plain `getModelContextWindow` is the obvious + * thing to reach for, and reaching for it here would silently skip the override. + */ +export function getModelContextWindowFromTable(model: string) { + return getKnownModelContextWindow(model) ?? ASSUMED_CONTEXT_WINDOW +} + +/** + * The context window stated for a model: the workspace's override + * (`context_window_per_model`, passed in so this module stays free of stores) + * when one is set, otherwise the built-in table. Undefined only when neither + * knows the model, which is when a consumer falls back to an assumed window. + */ +export function getConfiguredModelContextWindow( + provider: AIProvider, + model: string, + overrides: Record | undefined +): number | undefined { + return overrides?.[modelKey(provider, model)] ?? getKnownModelContextWindow(model) +} + +/** The window to budget against: the override, then the table, then the assumption. */ +export function getEffectiveModelContextWindow( + provider: AIProvider, + model: string, + overrides: Record | undefined +): number { + return getConfiguredModelContextWindow(provider, model, overrides) ?? ASSUMED_CONTEXT_WINDOW } /** diff --git a/frontend/src/lib/components/copilot/reasoningRegistry.ts b/frontend/src/lib/components/copilot/reasoningRegistry.ts index 242f6cb329..9915e9c42f 100644 --- a/frontend/src/lib/components/copilot/reasoningRegistry.ts +++ b/frontend/src/lib/components/copilot/reasoningRegistry.ts @@ -374,7 +374,8 @@ const ANTHROPIC_ALWAYS_THINKING = /fable|mythos/ /** * Disable token to forward when the user explicitly turns reasoning off on a * model that reasons *by default* — omitting the field would silently keep - * the default-on behavior. Undefined means omission is the correct off. + * the default-on behavior. Undefined means omission is the correct off. Every + * token is `'none'`: `requestsReasoning` reads that value as off. */ export function explicitOffToken(provider: AIProvider, model: string): ReasoningEffort | undefined { switch (reasoningProviderFamily(provider, model)) { @@ -434,6 +435,20 @@ export function resolveRequestReasoning( return undefined } +/** + * Whether a request carrying `effort`, as `resolveRequestReasoning` resolves it, + * has the model reason. An effort sent to a model the registry does not mark as + * reasoning-capable answers no: an explicit choice is always sent, but that does + * not make the model think. + */ +export function requestsReasoning( + provider: AIProvider, + model: string, + effort: ReasoningEffort | undefined +): boolean { + return !!effort && effort !== 'none' && supportsReasoning(provider, model) +} + export type ReasoningApiKind = 'anthropic' | 'responses' | 'completions' | 'deepseek' | 'mistral' /** diff --git a/frontend/src/lib/components/datatableAcl/PgAclEditor.svelte b/frontend/src/lib/components/datatableAcl/PgAclEditor.svelte new file mode 100644 index 0000000000..a3a765014a --- /dev/null +++ b/frontend/src/lib/components/datatableAcl/PgAclEditor.svelte @@ -0,0 +1,288 @@ + + +{#if acl.error} + {errorText(acl.error)} +{:else if !info} + Loading… +{:else} +
+ {#if info.clone} + + Read only: this data table is a clone, and its owners and grants stay as they were copied + from the data table it was cloned from. + + {:else if !info.editable} + + Read only: access is changed by the admins of the workspace that governs this data table, on + Windmill Enterprise Edition. + + {/if} + + {#if target.kind !== 'database'} +
+
+ Owner + + {target.kind === 'schema' + ? 'The role that owns the schema and everything already in it, except what belongs to an extension, which stays with the extension. Changing it also keeps the new owner in reach of what the current roles create here from then on; a role added afterwards is not covered.' + : 'The role that owns the table. Its owner may always read and write it, and is who ALTER and DROP answer to.'} + +
+ {#if info.editable} +
+ Role + Privileges + On + + + + + {#each grantRows as grant (grantKey(grant))} + {@const revokeScope = revokeScopeOf(grant)} + {@const revocable = revocablePrivileges(grant, target)} + {@const blocked = blockingSources(grant, revocable)} + {@const uncovered = uncoveredCreators(grant, info.roles)} + + {grant.grantee} + {grant.privileges.join(', ')} + + {grantScopeLabel(grant)} + {#if blocked.length > 0} + + from {blocked.join(', ')} + + {/if} + {#if uncovered.length > 0} + + · not for what {uncovered.join(', ')} + {uncovered.length === 1 ? 'creates' : 'create'} + + {/if} + + + + {#if info.editable && revokeScope && revocable.length > 0 && blocked.length === 0 && info.roles.includes(grant.grantee) && grant.grantee !== ADMIN_ROLE} + + + {/if} + + +{/if} + + (pending = undefined)} +> +
+ {#if pendingCoversObjects} + + The same privileges on several objects read as one row, and are revoked together. + + {/if} + {#each pending?.warnings ?? [] as warning (warning)} + {warning} + {/each} + + Runs against {datatable} in a single transaction: + + 
{(pending?.statements ?? []).join(';\n')};
+
+ diff --git a/frontend/src/lib/components/datatableAcl/PgGrantBuilder.svelte b/frontend/src/lib/components/datatableAcl/PgGrantBuilder.svelte new file mode 100644 index 0000000000..5e5119abb6 --- /dev/null +++ b/frontend/src/lib/components/datatableAcl/PgGrantBuilder.svelte @@ -0,0 +1,95 @@ + + +
+
+ GRANT + ({ value: p, label: p }))} + placeholder="privileges" + {disabled} + size="sm" + class="min-w-48" + /> + ON + ({ value: r, label: r }))} + placeholder="role" + {disabled} + size="sm" + class="w-40" + /> + +
+ {#if statement} + 
{statement}
+ {/if} +
diff --git a/frontend/src/lib/components/datatableAcl/aclScopes.test.ts b/frontend/src/lib/components/datatableAcl/aclScopes.test.ts new file mode 100644 index 0000000000..7c01836e89 --- /dev/null +++ b/frontend/src/lib/components/datatableAcl/aclScopes.test.ts @@ -0,0 +1,178 @@ +import { describe, expect, it } from 'vitest' +import type { AclGrant } from '$lib/gen' +import { + blockingSources, + grantKey, + groupGrants, + revocablePrivileges, + revokeScopeOf, + uncoveredCreators +} from './aclScopes' + +const table = (name: string) => ({ name, kind: 'TABLE' }) +const by = (role: string, privileges: string[], reachable = true) => ({ + role, + privileges, + reachable +}) +const byAdmin = (grant: Omit): AclGrant => ({ + ...grant, + sources: [by('admin', grant.privileges)] +}) + +describe('grantKey', () => { + it('tells apart a table and a function of the same name', () => { + const row = (object: { name: string; kind: string; args?: string }) => ({ + grantee: 'analytics', + privileges: ['SELECT'], + objects: [object], + sources: [by('admin', ['SELECT'])] + }) + expect(grantKey(row(table('orders')))).not.toBe( + grantKey(row({ name: 'orders', kind: 'FUNCTION', args: '' })) + ) + }) +}) + +describe('groupGrants', () => { + // A row's revoke names every object in it, so a row must only hold what one revoke may take. + it('folds the same privileges on objects of one kind, and nothing else', () => { + const grants: AclGrant[] = [ + { grantee: 'analytics', privileges: ['SELECT'], object: table('orders') }, + { grantee: 'analytics', privileges: ['SELECT'], object: table('salaries') }, + { grantee: 'operator', privileges: ['SELECT'], object: table('orders') }, + { grantee: 'analytics', privileges: ['INSERT', 'SELECT'], object: table('events') }, + { grantee: 'analytics', privileges: ['SELECT'], object: { name: 's', kind: 'SEQUENCE' } }, + { grantee: 'analytics', privileges: ['SELECT'], future: 'TABLES' }, + { grantee: 'analytics', privileges: ['USAGE'] } + ].map(byAdmin) + const rows = groupGrants(grants) + expect(rows.map((r) => [r.grantee, r.privileges, r.objects, r.future])).toEqual([ + ['analytics', ['SELECT'], [table('orders'), table('salaries')], undefined], + ['operator', ['SELECT'], [table('orders')], undefined], + ['analytics', ['INSERT', 'SELECT'], [table('events')], undefined], + ['analytics', ['SELECT'], [{ name: 's', kind: 'SEQUENCE' }], undefined], + ['analytics', ['SELECT'], [], 'TABLES'], + ['analytics', ['USAGE'], [], undefined] + ]) + }) + + // A revoke takes the row back from every source, so the row must name them all, with what each + // gave. + it('keeps every source of the grants it folds', () => { + const grants: AclGrant[] = [ + { + grantee: 'analytics', + privileges: ['SELECT'], + object: table('orders'), + sources: [by('admin', ['SELECT'])] + }, + { + grantee: 'analytics', + privileges: ['SELECT'], + object: table('salaries'), + sources: [by('admin', ['SELECT']), by('operator', ['SELECT'])] + } + ] + expect(groupGrants(grants)[0].sources).toEqual([ + by('admin', ['SELECT']), + by('operator', ['SELECT']) + ]) + }) +}) + +describe('revoke of a row', () => { + const row = (future?: string) => ({ + grantee: 'analytics', + privileges: ['SELECT'], + objects: [], + future, + sources: [by('admin', ['SELECT'])] + }) + + it('takes back only what the editor may revoke on the database', () => { + const database = { ...row(), privileges: ['CONNECT', 'CREATE'] } + expect(revocablePrivileges(database, { kind: 'database' })).toEqual(['CREATE']) + expect(revocablePrivileges(database, { kind: 'schema', schema: 'public' })).toEqual([ + 'CONNECT', + 'CREATE' + ]) + // Set database-wide, so not one a schema's revoke could take back either. + const schemasLater = { ...row('SCHEMAS'), privileges: ['CREATE'] } + expect(revocablePrivileges(schemasLater, { kind: 'database' })).toEqual([]) + }) + + it('maps default privileges to their scope, and refuses the ones it has none for', () => { + expect(revokeScopeOf(row())).toBe('target') + expect(revokeScopeOf(row('TABLES'))).toBe('future_tables') + expect(revokeScopeOf(row('TYPES'))).toBeUndefined() + expect(revokeScopeOf({ ...row(), objects: [{ name: 'mood', kind: 'TYPE' }] })).toBeUndefined() + }) + + // Postgres takes a grant back only through its source: offering the revoke would promise what + // the plan then refuses. But only the sources of what is revoked count: the catalog's CONNECT + // on the database comes from its owner, out of reach, and must not hold back a CREATE the + // editor granted. + it('is held back only by a source out of reach for what it takes', () => { + const database = { + ...row(), + privileges: ['CONNECT', 'CREATE'], + sources: [by('postgres', ['CONNECT'], false), by('admin', ['CREATE'])] + } + const revocable = revocablePrivileges(database, { kind: 'database' }) + expect(blockingSources(database, revocable)).toEqual([]) + expect(blockingSources(database, ['CONNECT'])).toEqual(['postgres']) + const partly = { + ...row('TABLES'), + sources: [by('admin', ['SELECT']), by('postgres', ['SELECT'], false)] + } + expect(blockingSources(partly, ['SELECT'])).toEqual(['postgres']) + }) + + // Whether a grant can be taken back depends on its object, so a row folding several objects is + // only revocable if each of its grants is. + it('is held back by a source out of reach on any of the objects it folds', () => { + const grants: AclGrant[] = [ + { + grantee: 'analytics', + privileges: ['SELECT'], + object: table('orders'), + sources: [by('admin', ['SELECT'])] + }, + { + grantee: 'analytics', + privileges: ['SELECT'], + object: table('salaries'), + sources: [by('admin', ['SELECT'], false)] + } + ] + const [folded] = groupGrants(grants) + expect(folded.objects).toHaveLength(2) + expect(blockingSources(folded, ['SELECT'])).toEqual(['admin']) + // Folding reads the grants, never rewrites them. + expect(grants[0].sources[0].reachable).toBe(true) + }) +}) + +describe('uncoveredCreators', () => { + // A default privilege binds only the creating roles it was granted for: a role added since is + // left out until the grant is made again. + it('names the roles a created-later row leaves out', () => { + const future = { + grantee: 'analytics', + privileges: ['SELECT'], + objects: [], + future: 'TABLES', + sources: [by('admin', ['SELECT']), by('analytics', ['SELECT'])] + } + expect(uncoveredCreators(future, ['admin', 'analytics', 'late'])).toEqual(['late']) + expect(uncoveredCreators({ ...future, future: undefined }, ['late'])).toEqual([]) + // Set by a role outside the catalog, it was never meant to cover the catalog's roles. + expect( + uncoveredCreators({ ...future, sources: [by('postgres', ['SELECT'], false)] }, [ + 'admin', + 'late' + ]) + ).toEqual([]) + }) +}) diff --git a/frontend/src/lib/components/datatableAcl/aclScopes.ts b/frontend/src/lib/components/datatableAcl/aclScopes.ts new file mode 100644 index 0000000000..d5b8ad9d20 --- /dev/null +++ b/frontend/src/lib/components/datatableAcl/aclScopes.ts @@ -0,0 +1,211 @@ +import type { AclGrant, AclSource, AclTarget } from '$lib/gen' + +/** The role a data table connects as without roles — `custom_instance_user` in Postgres. */ +export const ADMIN_ROLE = 'admin' + +/** Privileges Postgres accepts per kind of object. Mirrors the whitelist the backend validates + * against — a privilege missing here just cannot be built. */ +/** `CREATE` on a database is the right to create schemas in it, and the only database privilege + * handed out here: `CONNECT` is managed with the instance's role catalog. */ +export const DATABASE_PRIVILEGES = ['CREATE'] +export const SCHEMA_PRIVILEGES = ['USAGE', 'CREATE'] +export const TABLE_PRIVILEGES = [ + 'SELECT', + 'INSERT', + 'UPDATE', + 'DELETE', + 'TRUNCATE', + 'REFERENCES', + 'TRIGGER' +] +/** Postgres 17 and later only, so it is offered from what the server reports. */ +export const MAINTAIN_PRIVILEGE = 'MAINTAIN' +export const SEQUENCE_PRIVILEGES = ['USAGE', 'SELECT', 'UPDATE'] +export const FUNCTION_PRIVILEGES = ['EXECUTE'] + +export type AclScope = + | 'target' + | 'all_tables' + | 'all_sequences' + | 'all_functions' + | 'future_tables' + | 'future_sequences' + | 'future_functions' + +export type AclTargetKind = AclTarget['kind'] + +/** The scopes a target can grant on, in the order the builder offers them. */ +export function scopesOf(kind: AclTargetKind): { value: AclScope; label: string }[] { + if (kind === 'database') return [{ value: 'target', label: 'the database itself' }] + if (kind === 'table') return [{ value: 'target', label: 'this table' }] + return [ + { value: 'target', label: 'the schema itself' }, + { value: 'all_tables', label: 'all tables in it' }, + { value: 'all_sequences', label: 'all sequences in it' }, + { value: 'all_functions', label: 'all functions in it' }, + { value: 'future_tables', label: 'tables created later' }, + { value: 'future_sequences', label: 'sequences created later' }, + { value: 'future_functions', label: 'functions created later' } + ] +} + +export function privilegesOf( + scope: AclScope, + kind: AclTargetKind, + supportsMaintain = false +): string[] { + const tablePrivileges = supportsMaintain + ? [...TABLE_PRIVILEGES, MAINTAIN_PRIVILEGE] + : TABLE_PRIVILEGES + switch (scope) { + case 'target': + if (kind === 'database') return DATABASE_PRIVILEGES + return kind === 'schema' ? SCHEMA_PRIVILEGES : tablePrivileges + case 'all_tables': + case 'future_tables': + return tablePrivileges + case 'all_sequences': + case 'future_sequences': + return SEQUENCE_PRIVILEGES + case 'all_functions': + case 'future_functions': + return FUNCTION_PRIVILEGES + } +} + +/** What a statement built at this scope reads as, for the builder's own preview. */ +export function scopeSql(scope: AclScope, target: AclTarget, dbname?: string): string { + if (target.kind === 'database') return `DATABASE ${dbname ?? ''}`.trim() + const schema = target.schema + switch (scope) { + case 'target': + return target.kind === 'schema' ? `SCHEMA ${schema}` : `TABLE ${schema}.${target.table}` + case 'all_tables': + return `ALL TABLES IN SCHEMA ${schema}` + case 'all_sequences': + return `ALL SEQUENCES IN SCHEMA ${schema}` + case 'all_functions': + return `ALL FUNCTIONS IN SCHEMA ${schema}` + case 'future_tables': + return `TABLES (default privileges in ${schema})` + case 'future_sequences': + return `SEQUENCES (default privileges in ${schema})` + case 'future_functions': + return `FUNCTIONS (default privileges in ${schema})` + } +} + +/** One row of the grants table: the same privileges on several objects read as one line, since + * granting them per object is what `ON ALL TABLES` does. */ +export type GroupedGrant = { + grantee: string + privileges: string[] + objects: NonNullable[] + future?: string + /** Every role the row's grants come from, each once, with what it gave. */ + sources: AclSource[] +} + +export function groupGrants(grants: AclGrant[]): GroupedGrant[] { + const rows: GroupedGrant[] = [] + for (const grant of grants) { + const existing = grant.object + ? rows.find( + (r) => + r.grantee === grant.grantee && + r.future === grant.future && + r.objects[0]?.kind === grant.object?.kind && + r.privileges.join() === grant.privileges.join() + ) + : undefined + if (existing) { + existing.objects.push(grant.object!) + for (const source of grant.sources) { + const known = existing.sources.find((s) => s.role === source.role) + // Whether a role's grant can be taken back depends on the object it is on, so a row + // holds a source as reachable only if it is on every object the row folds. + if (known) { + known.reachable &&= source.reachable + known.privileges = [...new Set([...known.privileges, ...source.privileges])].sort() + } else { + existing.sources.push({ ...source, privileges: [...source.privileges] }) + } + } + } else { + rows.push({ + grantee: grant.grantee, + privileges: grant.privileges, + objects: grant.object ? [grant.object] : [], + future: grant.future, + sources: grant.sources.map((s) => ({ ...s, privileges: [...s.privileges] })) + }) + } + } + return rows +} + +/** The roles that gave some of `privileges` and that this data table's connection cannot act for. + * Only they can take those grants back, so a revoke of `privileges` is not offered. */ +export function blockingSources(grant: GroupedGrant, privileges: string[]): string[] { + return grant.sources + .filter((s) => !s.reachable && s.privileges.some((p) => privileges.includes(p))) + .map((s) => s.role) +} + +/** Which of `roles` a "created later" row granted for some of them does not cover. A default + * privilege binds only the creating roles it was granted for, so what the others create stays out + * of it. A row none of `roles` set — the instance's own, say — was never meant to cover them, and + * names none. */ +export function uncoveredCreators(grant: GroupedGrant, roles: string[]): string[] { + if (!grant.future || !grant.sources.some((s) => roles.includes(s.role))) return [] + return roles.filter((r) => !grant.sources.some((s) => s.role === r)) +} + +/** A row's identity. Two rows may share a grantee and an object name — a table `orders` and a + * function `orders()` — so the kind and the privileges are part of it too. */ +export function grantKey(grant: GroupedGrant): string { + return [ + grant.grantee, + grant.future ?? '', + grant.privileges.join(','), + ...grant.objects.map((o) => `${o.kind}:${o.name}(${o.args ?? ''})`) + ].join('|') +} + +/** The scope a revoke of this row takes, or `undefined` when the builder cannot express it — + * Postgres also records privileges on types, present and default, which nothing here grants and + * the API has no scope for. */ +export function revokeScopeOf(grant: GroupedGrant): AclScope | undefined { + if (!grant.future) return grant.objects.some((o) => o.kind === 'TYPE') ? undefined : 'target' + const scope = `future_${grant.future.toLowerCase()}` + return (['future_tables', 'future_sequences', 'future_functions'] as const).find( + (s) => s === scope + ) +} + +/** The privileges of a row a revoke may take back. On the database that is `CREATE` alone: + * `CONNECT` belongs to the role catalog, which would grant it again, and `TEMPORARY` is not one + * the editor hands out — a row holding only those has nothing to revoke here. A database's rows + * "created later" are default privileges set database-wide, which nothing here revokes. */ +export function revocablePrivileges(grant: GroupedGrant, target: AclTarget): string[] { + if (target.kind === 'database' && grant.objects.length === 0) { + if (grant.future) return [] + return grant.privileges.filter((p) => DATABASE_PRIVILEGES.includes(p)) + } + return grant.privileges +} + +/** How a row reads back: what it covers, in one phrase. */ +export function grantScopeLabel(grant: GroupedGrant): string { + if (grant.future) return `${grant.future.toLowerCase()} created later` + if (grant.objects.length === 1) { + const object = grant.objects[0] + // A routine's arguments are part of what it is, so two of the same name would otherwise + // read as one row twice. + const args = object.args !== undefined ? `(${object.args})` : '' + return `${object.kind.toLowerCase()} ${object.name}${args}` + } + if (grant.objects.length > 1) + return `${grant.objects.length} ${grant.objects[0].kind.toLowerCase()}s` + return 'itself' +} diff --git a/frontend/src/lib/components/datatableMigrationRole.test.ts b/frontend/src/lib/components/datatableMigrationRole.test.ts new file mode 100644 index 0000000000..6d136b8371 --- /dev/null +++ b/frontend/src/lib/components/datatableMigrationRole.test.ts @@ -0,0 +1,60 @@ +import { describe, test, expect } from 'vitest' +import { parseMigrationRole, withMigrationRole } from './datatableMigrationRole' + +describe('parseMigrationRole', () => { + test('reads every spelling the server accepts from the leading comment block', () => { + for (const line of [ + '-- role analyst', + '-- Role: analyst', + '-- role=analyst', + '-- role analyst;' + ]) { + expect(parseMigrationRole(`\n${line}\nBEGIN;\nEND;`)).toEqual({ + kind: 'role', + role: 'analyst' + }) + } + }) + + test('an annotation below BEGIN is not one', () => { + expect(parseMigrationRole('BEGIN;\n-- role analyst\nEND;')).toEqual({ kind: 'none' }) + }) + + test('a malformed attempt is an error, not the default', () => { + for (const line of [ + '-- role based access below', + '-- role', + '-- role:', + '-- role an;alytics' + ]) { + expect(parseMigrationRole(`${line}\nBEGIN;`)).toEqual({ kind: 'malformed', line }) + } + }) + + test('comments that do not start with the word role are ignored', () => { + expect(parseMigrationRole('-- roles analyst\n-- rolex\nBEGIN;')).toEqual({ kind: 'none' }) + }) +}) + +describe('withMigrationRole', () => { + test('leads above BEGIN, so the server reads it', () => { + const out = withMigrationRole('BEGIN;\nSELECT 1;\nEND;', 'analyst') + expect(out).toBe('-- role analyst\nBEGIN;\nSELECT 1;\nEND;') + }) + + test('replaces any attempt rather than stacking, malformed ones included', () => { + const out = withMigrationRole( + '-- Role: auditor\n-- role oops no\n-- keep me\nBEGIN;', + 'analyst' + ) + expect(out).toBe('-- role analyst\n-- keep me\nBEGIN;') + }) + + test('undefined strips the annotation, so it runs as admin', () => { + expect(withMigrationRole('-- role analyst\n\nBEGIN;\nEND;', undefined)).toBe('BEGIN;\nEND;') + }) + + test('refuses a name the server would refuse', () => { + expect(() => withMigrationRole('BEGIN;', 'bad;name')).toThrow() + }) +}) diff --git a/frontend/src/lib/components/datatableMigrationRole.ts b/frontend/src/lib/components/datatableMigrationRole.ts new file mode 100644 index 0000000000..5475bf2fd7 --- /dev/null +++ b/frontend/src/lib/components/datatableMigrationRole.ts @@ -0,0 +1,70 @@ +import { isDatatableRoleName } from './dbTypes' + +/** + * A migration carries the data table role it runs as in its own SQL, as a `-- role ` + * annotation. There is no separate field: the annotation is what the server reads, and keeping + * it in the SQL is what lets it survive a `wmill sync` round-trip. + * + * Mirrors `SqlAnnotations::datatable_role` on the backend. It is only read from the leading + * comment block, so an annotation below `BEGIN;` is ignored and the migration runs as admin. A + * leading comment whose first word is `role` is an annotation attempt, and a malformed one is an + * error there, so it is one here too. + */ + +export type MigrationRole = + | { kind: 'none' } + | { kind: 'role'; role: string } + | { kind: 'malformed'; line: string } + +/** The body of a leading comment line that attempts a role annotation, or undefined. */ +function roleAttempt(line: string): string | undefined { + if (!line.startsWith('--')) return undefined + const body = line.slice(2).trimStart() + if (body.slice(0, 4).toLowerCase() !== 'role') return undefined + const after = body.slice(4) + if (after !== '' && !/^[\s:=]/.test(after)) return undefined + return after +} + +function parseAttempt(after: string): string | undefined { + let rest = after.trimStart() + if (rest.startsWith(':') || rest.startsWith('=')) rest = rest.slice(1) + const tokens = rest.split(/\s+/).filter((t) => t !== '') + if (tokens.length !== 1) return undefined + const role = tokens[0].endsWith(';') ? tokens[0].slice(0, -1) : tokens[0] + return isDatatableRoleName(role) ? role : undefined +} + +export function parseMigrationRole(sql: string): MigrationRole { + for (const raw of sql.split('\n')) { + const line = raw.trim() + if (line === '') continue + if (!line.startsWith('--')) break + const after = roleAttempt(line) + if (after === undefined) continue + const role = parseAttempt(after) + return role === undefined ? { kind: 'malformed', line } : { kind: 'role', role } + } + return { kind: 'none' } +} + +/** + * `sql` declaring `role`: any role annotation attempt in the leading comment block is removed, + * and `-- role ` is prepended above everything, or nothing when `role` is undefined. + */ +export function withMigrationRole(sql: string, role: string | undefined): string { + if (role !== undefined && !isDatatableRoleName(role)) { + throw new Error(`Invalid data table role '${role}'`) + } + const lines = sql.split('\n') + const kept: string[] = [] + let i = 0 + for (; i < lines.length; i++) { + const line = lines[i].trim() + if (line !== '' && !line.startsWith('--')) break + if (roleAttempt(line) === undefined) kept.push(lines[i]) + } + const rest = [...kept, ...lines.slice(i)] + while (rest.length > 0 && rest[0].trim() === '') rest.shift() + return role === undefined ? rest.join('\n') : [`-- role ${role}`, ...rest].join('\n') +} diff --git a/frontend/src/lib/components/datatableUsableRoles.test.ts b/frontend/src/lib/components/datatableUsableRoles.test.ts new file mode 100644 index 0000000000..cca4ab7607 --- /dev/null +++ b/frontend/src/lib/components/datatableUsableRoles.test.ts @@ -0,0 +1,29 @@ +import { describe, expect, it, vi } from 'vitest' + +const request = vi.fn() +vi.mock('$lib/gen/core/request', () => ({ request: (...args: unknown[]) => request(...args) })) +vi.mock('$lib/gen', () => ({ OpenAPI: {} })) +vi.mock('$lib/cloud', () => ({ isCloudHosted: () => false })) + +import { listUsableDatatableRoles } from './datatableUsableRoles' + +const refuse = (body: string) => () => + Promise.reject(Object.assign(new Error('Bad Request'), { body })) + +describe('listUsableDatatableRoles', () => { + // The refusal is recognised by its sentence, so rewording it on one side alone would turn every + // data table on a build without the Enterprise Edition into a failed lookup rather than one + // that is not under roles. + it('reads the enterprise refusal as not under roles, and rethrows anything else', async () => { + request.mockImplementation(refuse('Data table roles are a Windmill Enterprise Edition feature')) + expect(await listUsableDatatableRoles('ws', 'main')).toEqual({ + permissioned: false, + roles: [], + default_role: 'admin' + }) + + request.mockImplementation(refuse('Data table not found')) + const rethrown = await listUsableDatatableRoles('ws', 'main').catch((e) => e) + expect(rethrown).toBeInstanceOf(Error) + }) +}) diff --git a/frontend/src/lib/components/datatableUsableRoles.ts b/frontend/src/lib/components/datatableUsableRoles.ts new file mode 100644 index 0000000000..84e52f29a8 --- /dev/null +++ b/frontend/src/lib/components/datatableUsableRoles.ts @@ -0,0 +1,44 @@ +import { OpenAPI, type ListUsableDatatableRolesResponse } from '$lib/gen' +import { request } from '$lib/gen/core/request' +import { isCloudHosted } from '$lib/cloud' +import { ADMIN_DATATABLE_ROLE } from './dbTypes' + +// `datatable_roles_unavailable` on the server, which is a plain 400: rewording it there without +// here makes every role picker on a non-Enterprise build fail instead of reading "not under roles". +const ROLES_UNAVAILABLE = 'Data table roles are a Windmill Enterprise Edition feature' + +const NOT_UNDER_ROLES: ListUsableDatatableRolesResponse = { + permissioned: false, + roles: [], + default_role: ADMIN_DATATABLE_ROLE +} + +/** + * The roles the caller may connect as on a data table. Cloud has no instance database, so no data + * table there is under roles, and none of the role pickers show. Without the Enterprise Edition + * every roles route refuses, which reads the same way: the data table is then used the way it was + * before roles, and one that is under roles is refused when something connects to it. + */ +export async function listUsableDatatableRoles( + workspace: string, + datatableName: string +): Promise { + if (isCloudHosted()) return NOT_UNDER_ROLES + try { + // The generated client encodes path params with `encodeURI`, which leaves a '?' in a + // data table name created before names were restricted to cut the path short. + return await request( + { ...OpenAPI, ENCODE_PATH: encodeURIComponent }, + { + method: 'GET', + url: '/w/{workspace}/workspaces/datatable_usable_roles/{datatable_name}', + path: { workspace, datatable_name: datatableName } + } + ) + } catch (e) { + const body = (e as { body?: unknown })?.body + const detail = `${typeof body === 'string' ? body : JSON.stringify(body ?? '')} ${(e as Error)?.message ?? e}` + if (detail.includes(ROLES_UNAVAILABLE)) return NOT_UNDER_ROLES + throw e + } +} diff --git a/frontend/src/lib/components/dbManagerDrawerModel.svelte.ts b/frontend/src/lib/components/dbManagerDrawerModel.svelte.ts index 59345c6ae7..890635dcf8 100644 --- a/frontend/src/lib/components/dbManagerDrawerModel.svelte.ts +++ b/frontend/src/lib/components/dbManagerDrawerModel.svelte.ts @@ -5,7 +5,7 @@ import { isDbType } from './dbTypes' /** * Single URL param `dbm` encodes the full DB manager state: - * firstSegment~path~schema.table + * firstSegment~path~schema.table~role=name * * firstSegment: * datatable – database with datatable:// resource (resourceType always postgresql) @@ -26,28 +26,46 @@ import { isDbType } from './dbTypes' * datatable~main~.customers (schema "public" implied) * ducklake~main~.orders (schema "main" implied) * postgresql~$res:u/user/my_pg~public.customers + * datatable~main~.customers~role=analyst + * datatable~main~role=analyst (no schema/table selected) + * + * role=name (last segment, optional, data tables only): the data table role to connect as. + * Omitted means the data table's default role. A trailing segment starting with `role=` is always + * the role, whatever follows. The name is kept as written, even when invalid (a `.` included), so + * the connection refuses it visibly instead of falling back to the default. */ const dbManagerSchema = z.object({ dbm: z.string().nullable() }) -interface ParsedDbm { +export interface ParsedDbm { type: 'database' | 'datatable' | 'ducklake' path: string resType?: string schema?: string table?: string + role?: string } -function parseDbm(raw: unknown): ParsedDbm | null { +const ROLE_SEGMENT_PREFIX = 'role=' + +function isRoleSegment(segment: string | undefined): segment is string { + return !!segment && segment.startsWith(ROLE_SEGMENT_PREFIX) +} + +export function parseDbm(raw: unknown): ParsedDbm | null { if (!raw || typeof raw !== 'string') return null const parts = raw.split('~') if (parts.length < 2 || !parts[1]) return null const firstSeg = parts[0] const path = parts[1] - const schemaTable = parts[2] ?? '' + const rest = parts.slice(2) + const role = isRoleSegment(rest.at(-1)) + ? rest.pop()!.slice(ROLE_SEGMENT_PREFIX.length) + : undefined + const schemaTable = rest[0] ?? '' let type: ParsedDbm['type'] let resType: string | undefined @@ -81,12 +99,12 @@ function parseDbm(raw: unknown): ParsedDbm | null { schema = defaultSchemas[type] } - return { type, path, resType, schema, table } + return { type, path, resType, schema, table, role: type === 'datatable' ? role : undefined } } const defaultSchemas: Record = { datatable: 'public', ducklake: 'main' } -function buildDbm(p: ParsedDbm): string { +export function buildDbm(p: ParsedDbm): string { const firstSeg = p.type === 'database' ? p.resType! : p.type const schema = p.schema === defaultSchemas[p.type] ? undefined : p.schema let schemaTable = '' @@ -97,7 +115,12 @@ function buildDbm(p: ParsedDbm): string { } else if (schema) { schemaTable = `${schema}.` } - return schemaTable ? `${firstSeg}~${p.path}~${schemaTable}` : `${firstSeg}~${p.path}` + const segments = [firstSeg, p.path] + if (schemaTable) segments.push(schemaTable) + if (p.type === 'datatable' && p.role !== undefined) { + segments.push(`${ROLE_SEGMENT_PREFIX}${p.role}`) + } + return segments.join('~') } export interface DbManagerUriState { @@ -105,6 +128,8 @@ export interface DbManagerUriState { readonly effectiveInput: DbInput | undefined readonly isDatatableInput: boolean selectedDatatable: string | undefined + /** The data table role the drawer connects as; undefined means its default. */ + selectedRole: string | undefined selectedSchema: string | undefined selectedTable: string | undefined readonly open: boolean @@ -137,6 +162,7 @@ export function useDbManagerUriState(): DbManagerUriState { type: 'database' as const, resourceType: resType as DbType, resourcePath: parsed.type === 'datatable' ? `datatable://${parsed.path}` : parsed.path, + role: parsed.role, specificSchema: parsed.schema, specificTable: parsed.table } @@ -163,6 +189,7 @@ export function useDbManagerUriState(): DbManagerUriState { type: isDatatable ? 'datatable' : 'database', path: isDatatable ? nInput.resourcePath.slice('datatable://'.length) : nInput.resourcePath, resType: isDatatable ? undefined : nInput.resourceType, + role: isDatatable ? nInput.role : undefined, schema: nInput.specificSchema, table: nInput.specificTable }) @@ -194,7 +221,14 @@ export function useDbManagerUriState(): DbManagerUriState { return parsed?.type === 'datatable' ? parsed.path : undefined }, set selectedDatatable(v: string | undefined) { - if (v) updateField({ path: v }) + // A role belongs to one data table, so it cannot carry over to another. + if (v) updateField({ path: v, role: undefined }) + }, + get selectedRole() { + return parsed?.role + }, + set selectedRole(v: string | undefined) { + updateField({ role: v }) }, get selectedSchema() { return parsed?.schema diff --git a/frontend/src/lib/components/dbManagerRole.test.ts b/frontend/src/lib/components/dbManagerRole.test.ts new file mode 100644 index 0000000000..6c45d0ed30 --- /dev/null +++ b/frontend/src/lib/components/dbManagerRole.test.ts @@ -0,0 +1,67 @@ +import { describe, expect, it } from 'vitest' +import { buildDbm, parseDbm } from './dbManagerDrawerModel.svelte' +import { schemaCacheKey } from './dbSchemaCache' +import { datatableReference, type DbInput } from './dbTypes' + +describe('dbm role segment', () => { + it('round-trips a role, with and without a table', () => { + for (const dbm of ['datatable~main~.orders~role=p4_analytics', 'datatable~main~role=p4-op']) { + expect(buildDbm(parseDbm(dbm)!)).toBe(dbm) + } + expect(parseDbm('datatable~main~sales.orders~role=analyst')).toMatchObject({ + path: 'main', + schema: 'sales', + table: 'orders', + role: 'analyst' + }) + }) + + it('reads a link without a role as the default role', () => { + const parsed = parseDbm('datatable~main~.orders')! + expect(parsed.role).toBeUndefined() + expect(parsed).toMatchObject({ schema: 'public', table: 'orders' }) + expect(buildDbm(parsed)).toBe('datatable~main~.orders') + }) + + it('keeps an invalid role as written, so the connection refuses it', () => { + expect(parseDbm('datatable~main~role=a;b')?.role).toBe('a;b') + // A dot does not turn it into a schema.table selection read as the default role. + expect(parseDbm('datatable~main~role=bad.name')).toMatchObject({ + role: 'bad.name', + schema: undefined, + table: undefined + }) + }) +}) + +describe('connecting as a role', () => { + const input = (role?: string): DbInput => ({ + type: 'database', + resourceType: 'postgresql', + resourcePath: 'datatable://main', + role + }) + + // What `getDatabaseArg` builds every DB manager connection from. + it('appends the role to the data table reference', () => { + expect(datatableReference('main', 'p4_analytics')).toBe('datatable://main?role=p4_analytics') + expect(datatableReference('main', undefined)).toBe('datatable://main') + }) + + it('never appends a role to a name containing ?', () => { + // The server reads such a whole reference as the stored name first, so `?role=` would + // be taken as part of the name or refused instead of picking the role. + expect(datatableReference('legacy?x', undefined)).toBe('datatable://legacy?x') + expect(() => datatableReference('sales?role=analytics', 'admin')).toThrow(/'\?' in its name/) + }) + + it('refuses a role name the server would not accept', () => { + expect(() => datatableReference('main', 'a&role=admin')).toThrow(/Invalid data table role/) + expect(() => datatableReference('main', '')).toThrow(/Invalid data table role/) + }) + + it('keys the schema cache by role', () => { + expect(schemaCacheKey('ws', input('a'))).not.toBe(schemaCacheKey('ws', input('b'))) + expect(schemaCacheKey('ws', input('a'))).not.toBe(schemaCacheKey('ws', input())) + }) +}) diff --git a/frontend/src/lib/components/dbOps.ts b/frontend/src/lib/components/dbOps.ts index 5005cc931c..dbe507d13f 100644 --- a/frontend/src/lib/components/dbOps.ts +++ b/frontend/src/lib/components/dbOps.ts @@ -8,7 +8,8 @@ import { runScriptAndPollResult } from './jobs/utils' import { writingJobOptions } from './jobs/writingJob' import type { DBSchema, SQLSchema } from '$lib/stores' import { stringifySchema } from './copilot/lib' -import type { DbInput, DbType } from './dbTypes' +import { datatableReference, type DbInput, type DbType } from './dbTypes' +import { withMigrationRole } from './datatableMigrationRole' import { assert } from '$lib/utils' import { WorkspaceService } from '$lib/gen' import { pendingMigrations } from './workspaceSettings/datatableMigrationUtils' @@ -23,6 +24,7 @@ import { transformSnowflakeForeignKeys, type RawForeignKey } from './apps/components/display/dbtable/queries/relationalKeys' +import { groupForeignKeyRows, type DbRelation, type RawAllForeignKeyRow } from './dbRelations' export type IDbTableOps = { dbType: DbType @@ -70,7 +72,9 @@ export function dbTableOpsWithPreviewScripts({ }): IDbTableOps { const dbType = getDbType(input) const language = getLanguageByResourceType(dbType) - const dbArg = getDatabaseArg(input) + // Built per call: an invalid role throws there, as that operation's error, rather than while + // the manager renders. + const dbArg = () => getDatabaseArg(input) const ducklake = input.type === 'ducklake' ? input.ducklake : undefined function makeMarker(op: string, payload: Record): string { @@ -91,7 +95,7 @@ export function dbTableOpsWithPreviewScripts({ }) const result = await runScriptAndPollResult({ workspace, - requestBody: { args: { ...dbArg, quicksearch }, language, content, tag } + requestBody: { args: { ...dbArg(), quicksearch }, language, content, tag } }) const count = result?.[0].count as number return count @@ -106,7 +110,7 @@ export function dbTableOpsWithPreviewScripts({ }) let items = (await runScriptAndPollResult({ workspace, - requestBody: { args: { ...dbArg, ...params }, language, content, tag } + requestBody: { args: { ...dbArg(), ...params }, language, content, tag } })) as unknown[] if (!items || !Array.isArray(items)) { throw 'items is not an array' @@ -123,7 +127,7 @@ export function dbTableOpsWithPreviewScripts({ { workspace, requestBody: { - args: { ...dbArg, value_to_update: newValue, ...values }, + args: { ...dbArg(), value_to_update: newValue, ...values }, language, content, tag @@ -135,14 +139,14 @@ export function dbTableOpsWithPreviewScripts({ onDelete: async ({ values }) => { const content = makeMarker('DELETE', { table: tableKey, columns: colDefs }) await runScriptAndPollResult( - { workspace, requestBody: { args: { ...dbArg, ...values }, language, content, tag } }, + { workspace, requestBody: { args: { ...dbArg(), ...values }, language, content, tag } }, writingJobOptions ) }, onInsert: async ({ values }) => { const content = makeMarker('INSERT', { table: tableKey, columns: colDefs }) await runScriptAndPollResult( - { workspace, requestBody: { args: { ...dbArg, ...values }, language, content, tag } }, + { workspace, requestBody: { args: { ...dbArg(), ...values }, language, content, tag } }, writingJobOptions ) } @@ -246,6 +250,7 @@ export type IDbSchemaOps = { previewAlterSql: (params: { values: AlterTableValues; schema?: string }) => Promise onCreateSchema: (params: { schema: string }) => Promise onDeleteSchema: (params: { schema: string }) => Promise + onRenameSchema: (params: { schema: string; newSchema: string }) => Promise onFetchTableEditorDefinition: (params: { table: string schema?: string @@ -255,6 +260,10 @@ export type IDbSchemaOps = { table: string schema?: string }) => Promise + /** Every foreign key of the database in one query, for the schema diagram. + * PostgreSQL only — the other databases either don't expose foreign keys or + * can only be asked one table at a time. */ + onFetchAllForeignKeys: () => Promise } /** Thrown by a schema op when the user declines the out-of-order run warning. @@ -283,7 +292,8 @@ export function dbSchemaOpsWithPreviewScripts({ tag?: string }): IDbSchemaOps { const dbType = getDbType(input) - const dbArg = getDatabaseArg(input) + // Built per call, for the same reason as in the table ops above. + const dbArg = () => getDatabaseArg(input) const language = getLanguageByResourceType(dbType) const ducklake = input.type === 'ducklake' ? input.ducklake : undefined @@ -293,6 +303,8 @@ export function dbSchemaOpsWithPreviewScripts({ input.type === 'database' && input.resourcePath.startsWith('datatable://') ? input.resourcePath.slice('datatable://'.length) : undefined + // A migration declaring no role runs as admin, whatever role the manager connects as. + const migrationRole = input.type === 'database' ? (input.role ?? input.migrationRole) : undefined function makeMarker(op: string, payload: Record): string { if (ducklake) payload.ducklake = ducklake @@ -359,7 +371,7 @@ export function dbSchemaOpsWithPreviewScripts({ : undefined if (!datatableName || !status?.enabled) { await runScriptAndPollResult( - { workspace, requestBody: { args: dbArg, content, language, tag } }, + { workspace, requestBody: { args: dbArg(), content, language, tag } }, writingJobOptions ) return @@ -373,12 +385,16 @@ export function dbSchemaOpsWithPreviewScripts({ throw new MigrationRunCancelled() } } - const codeUp = wrapMigration(await expandMarker(workspace, language, content)) + // Wrapped before annotating: the annotation must lead, above `BEGIN;`. + const codeUp = withMigrationRole( + wrapMigration(await expandMarker(workspace, language, content)), + migrationRole + ) // Down migrations are only generated for Postgres for now. let codeDown: string | undefined if (downContent && dbType === 'postgresql') { const downSql = (await expandMarker(workspace, language, downContent)).trim() - if (downSql) codeDown = wrapMigration(downSql) + if (downSql) codeDown = withMigrationRole(wrapMigration(downSql), migrationRole) } const created = await WorkspaceService.createDatatableMigration({ workspace, @@ -415,7 +431,7 @@ export function dbSchemaOpsWithPreviewScripts({ const fkContent = makeMarker('FOREIGN_KEYS', { table, schema }) const fkResult = await runScriptAndPollResult({ workspace, - requestBody: { args: dbArg, content: fkContent, language, tag } + requestBody: { args: dbArg(), content: fkContent, language, tag } }) let rawForeignKeys: RawForeignKey[] @@ -443,6 +459,16 @@ export function dbSchemaOpsWithPreviewScripts({ return [] } + async function fetchAllForeignKeys(): Promise { + if (dbType !== 'postgresql') return [] + const content = makeMarker('ALL_FOREIGN_KEYS', {}) + const rows = (await runScriptAndPollResult({ + workspace, + requestBody: { args: dbArg(), content, language, tag } + })) as RawAllForeignKeyRow[] + return Array.isArray(rows) ? groupForeignKeyRows(rows) : [] + } + return { onDelete: async ({ tableKey, schema }) => { const content = makeMarker('DROP_TABLE', { table: tableKey, schema }) @@ -501,7 +527,13 @@ export function dbSchemaOpsWithPreviewScripts({ const downContent = makeMarker('CREATE_SCHEMA', { schema }) await applyDdl(migrationName('drop_schema', schema), content, downContent) }, + onRenameSchema: async ({ schema, newSchema }) => { + const content = makeMarker('RENAME_SCHEMA', { schema, new_schema: newSchema }) + const downContent = makeMarker('RENAME_SCHEMA', { schema: newSchema, new_schema: schema }) + await applyDdl(migrationName('rename_schema', schema), content, downContent) + }, onFetchForeignKeys: fetchForeignKeys, + onFetchAllForeignKeys: fetchAllForeignKeys, onFetchTableEditorDefinition: async ({ table, schema, colDefs }) => { const foreignKeys = await fetchForeignKeys({ table, schema }) let pk_constraint_name: string | undefined @@ -512,7 +544,7 @@ export function dbSchemaOpsWithPreviewScripts({ const pkContent = makeMarker('PRIMARY_KEY_CONSTRAINT', { table, schema }) const pkResult = (await runScriptAndPollResult({ workspace, - requestBody: { args: dbArg, content: pkContent, language, tag } + requestBody: { args: dbArg(), content: pkContent, language, tag } })) as { constraint_name?: string; CONSTRAINT_NAME?: string }[] if (pkResult && Array.isArray(pkResult) && pkResult.length > 0) { @@ -611,7 +643,9 @@ export function getDefaultDbTag(input: DbInput): string { export function getDatabaseArg(input: DbInput | undefined) { if (input?.type === 'database') { if (input.resourcePath.startsWith('datatable://')) { - return { database: input.resourcePath } + return { + database: datatableReference(input.resourcePath.slice('datatable://'.length), input.role) + } } else { return { database: '$res:' + input.resourcePath } } diff --git a/frontend/src/lib/components/dbRelations.test.ts b/frontend/src/lib/components/dbRelations.test.ts new file mode 100644 index 0000000000..ad5d63a43e --- /dev/null +++ b/frontend/src/lib/components/dbRelations.test.ts @@ -0,0 +1,81 @@ +import { describe, expect, it } from 'vitest' +import { buildRelationIndex, groupForeignKeyRows, type RawAllForeignKeyRow } from './dbRelations' + +function row(over: Partial): RawAllForeignKeyRow { + return { + fk_constraint_name: 'fk', + source_schema: 'shop', + source_table: 'orders', + source_column: 'customer_id', + target_schema: 'shop', + target_table: 'customers', + target_column: 'id', + ordinal: 1, + ...over + } +} + +describe('groupForeignKeyRows', () => { + it('pairs a composite key by ordinal, whatever order the rows arrive in', () => { + const relations = groupForeignKeyRows([ + row({ + fk_constraint_name: 'returns_order_item_fkey', + source_table: 'returns', + source_column: 'sku', + target_table: 'order_items', + target_column: 'sku', + ordinal: 2 + }), + row({ + fk_constraint_name: 'returns_order_item_fkey', + source_table: 'returns', + source_column: 'order_id', + target_table: 'order_items', + target_column: 'order_id', + ordinal: 1 + }) + ]) + + expect(relations).toHaveLength(1) + expect(relations[0].from.columns).toEqual(['order_id', 'sku']) + expect(relations[0].to.columns).toEqual(['order_id', 'sku']) + }) + + it('keeps same-named constraints on different tables apart', () => { + const relations = groupForeignKeyRows([ + row({ fk_constraint_name: 'owner_fkey', source_table: 'orders' }), + row({ fk_constraint_name: 'owner_fkey', source_table: 'addresses' }) + ]) + expect(relations.map((r) => r.from.table).sort()).toEqual(['addresses', 'orders']) + }) +}) + +describe('buildRelationIndex', () => { + const index = buildRelationIndex( + groupForeignKeyRows([ + row({ source_table: 'orders', source_column: 'customer_id', target_column: 'id' }), + row({ + fk_constraint_name: 'items_order_fkey', + source_table: 'order_items', + source_column: 'order_id', + target_table: 'orders', + target_column: 'id' + }) + ]) + ) + + it('links a column only to the column it is actually paired with', () => { + // `order_items.order_id` points at `orders.id`, not at the same-named + // column of every other table taking part in a relation. + expect([...(index.linkedColumns.get('shop.order_items.order_id') ?? [])]).toEqual([ + 'shop.orders.id' + ]) + }) + + it('relates tables in both directions', () => { + expect([...(index.relatedTables.get('shop.orders') ?? [])].sort()).toEqual([ + 'shop.customers', + 'shop.order_items' + ]) + }) +}) diff --git a/frontend/src/lib/components/dbRelations.ts b/frontend/src/lib/components/dbRelations.ts new file mode 100644 index 0000000000..eb6451057e --- /dev/null +++ b/frontend/src/lib/components/dbRelations.ts @@ -0,0 +1,137 @@ +/** One row of the `ALL_FOREIGN_KEYS` marker: a single referencing column. */ +export type RawAllForeignKeyRow = { + fk_constraint_name: string + source_schema: string + source_table: string + source_column: string + target_schema: string + target_table: string + target_column: string + ordinal: number | string +} + +export type DbTableRef = { schema: string; table: string } + +/** A foreign key, with its columns paired to the ones they point at by index. */ +export type DbRelation = { + constraint: string + from: DbTableRef & { columns: string[] } + to: DbTableRef & { columns: string[] } +} + +export function tableKeyOf(ref: DbTableRef): string { + return `${ref.schema}.${ref.table}` +} + +export function columnKeyOf(ref: DbTableRef, column: string): string { + return columnKeyIn(tableKeyOf(ref), column) +} + +export function columnKeyIn(tableKey: string, column: string): string { + return `${tableKey}.${column}` +} + +/** Groups the flat rows into one relation per constraint, keeping the column + * pairs in the constraint's declaration order (the query's `ordinal`). */ +export function groupForeignKeyRows(rows: RawAllForeignKeyRow[]): DbRelation[] { + const byConstraint = new Map() + + for (const raw of rows) { + const row = lowercaseKeys(raw) + if (!row.source_table || !row.target_table) continue + const key = `${row.source_schema}.${row.source_table}.${row.fk_constraint_name}` + let entry = byConstraint.get(key) + if (!entry) { + entry = { + relation: { + constraint: row.fk_constraint_name, + from: { schema: row.source_schema, table: row.source_table, columns: [] }, + to: { schema: row.target_schema, table: row.target_table, columns: [] } + }, + pairs: [] + } + byConstraint.set(key, entry) + } + entry.pairs.push(row) + } + + return Array.from(byConstraint.values()).map(({ relation, pairs }) => { + pairs.sort((a, b) => Number(a.ordinal) - Number(b.ordinal)) + relation.from.columns = pairs.map((p) => p.source_column) + relation.to.columns = pairs.map((p) => p.target_column) + return relation + }) +} + +function lowercaseKeys(row: RawAllForeignKeyRow): RawAllForeignKeyRow { + const out: any = {} + for (const key of Object.keys(row)) out[key.toLowerCase()] = (row as any)[key] + return out as RawAllForeignKeyRow +} + +/** Everything the diagram needs to answer "what lights up when this is hovered". + * Built once per fetched relation set, then read on every hover. */ +export type RelationIndex = { + /** Tables reachable from a table by one foreign key, either direction. */ + relatedTables: Map> + /** Columns a column is paired with by a foreign key, either direction. */ + linkedColumns: Map> + /** Every column taking part in a relation of a table, keyed by table. Both + * the table's own referencing/referenced columns and the far ends. */ + relatedColumns: Map> + /** Columns that reference another table — rendered with a link marker. */ + referencingColumns: Set + /** Columns another table points at. */ + referencedColumns: Set + /** The table a column key belongs to. Both parts are dot-joined names, so + * splitting a column key back apart is not safe. */ + columnTable: Map +} + +export function buildRelationIndex(relations: DbRelation[]): RelationIndex { + const index: RelationIndex = { + relatedTables: new Map(), + linkedColumns: new Map(), + relatedColumns: new Map(), + referencingColumns: new Set(), + referencedColumns: new Set(), + columnTable: new Map() + } + + for (const relation of relations) { + const fromTable = tableKeyOf(relation.from) + const toTable = tableKeyOf(relation.to) + + add(index.relatedTables, fromTable, toTable) + add(index.relatedTables, toTable, fromTable) + + const pairCount = Math.min(relation.from.columns.length, relation.to.columns.length) + for (let i = 0; i < pairCount; i++) { + const fromColumn = columnKeyOf(relation.from, relation.from.columns[i]) + const toColumn = columnKeyOf(relation.to, relation.to.columns[i]) + + index.referencingColumns.add(fromColumn) + index.referencedColumns.add(toColumn) + index.columnTable.set(fromColumn, fromTable) + index.columnTable.set(toColumn, toTable) + + add(index.linkedColumns, fromColumn, toColumn) + add(index.linkedColumns, toColumn, fromColumn) + + // Both ends of the pair under both tables: hovering either table has to + // light its own column and the one it is paired with. + for (const table of [fromTable, toTable]) { + add(index.relatedColumns, table, fromColumn) + add(index.relatedColumns, table, toColumn) + } + } + } + + return index +} + +function add(map: Map>, key: string, value: string) { + let set = map.get(key) + if (!set) map.set(key, (set = new Set())) + set.add(value) +} diff --git a/frontend/src/lib/components/dbSchemaCache.ts b/frontend/src/lib/components/dbSchemaCache.ts new file mode 100644 index 0000000000..0421c0c033 --- /dev/null +++ b/frontend/src/lib/components/dbSchemaCache.ts @@ -0,0 +1,21 @@ +import type { DbInput } from './dbTypes' + +/** What identifies a database's schema, role included: two roles on one data table may reach + * different schemas, so they cannot share a cache entry. Never throws, since it keys derived + * state; the connection itself is what refuses an invalid role. */ +export function getDbSchemasPath(input: DbInput): string { + switch (input.type) { + case 'database': + return input.role !== undefined && input.resourcePath.startsWith('datatable://') + ? `${input.resourcePath}?role=${input.role}` + : input.resourcePath + case 'ducklake': + return 'ducklake://' + input.ducklake + } +} + +/** Scoped by the acting workspace: a data table of the same name can exist in both the nav and + * the acting workspace, and one's schema must not be reused for the other. */ +export function schemaCacheKey(workspace: string | undefined, input: DbInput): string { + return `${workspace}:${getDbSchemasPath(input)}` +} diff --git a/frontend/src/lib/components/dbTypes.ts b/frontend/src/lib/components/dbTypes.ts index 6a85617110..e6d3af12e8 100644 --- a/frontend/src/lib/components/dbTypes.ts +++ b/frontend/src/lib/components/dbTypes.ts @@ -3,6 +3,12 @@ export type DbInput = type: 'database' resourceType: DbType resourcePath: string + /** The data table role to connect as; the data table's default when unset. Only + * meaningful for a `datatable://` path. */ + role?: string + /** The role migrations written through this input declare when `role` is unset. A + * migration declaring none runs as admin, not as the role the manager connects as. */ + migrationRole?: string specificSchema?: string specificTable?: string } @@ -23,3 +29,47 @@ export const dbTypes = [ 'duckdb' ] as const export const isDbType = (str?: string): str is DbType => !!str && dbTypes.includes(str as DbType) + +/** The role every data table has: the one it connects as when it is not under roles. */ +export const ADMIN_DATATABLE_ROLE = 'admin' + +/** What the server accepts in `-- role ` and `?role=`. */ +export function isDatatableRoleName(name: string): boolean { + return /^[A-Za-z0-9_-]{1,63}$/.test(name) +} + +/** Whether a role can be named in a reference to this data table. A name stored before names were + * restricted may contain `?`, and the server reads such a whole reference as that name first, so + * `?role=` after it would be taken as part of the name or refused. */ +export function datatableNameTakesRole(name: string): boolean { + return !name.includes('?') +} + +/** The `migrationRole` of a data table that cannot name a role in its reference: it connects as + * its default role, which its migrations must then declare. */ +export function defaultMigrationRole( + name: string, + permissioned: boolean | undefined, + defaultRole: string | undefined +): string | undefined { + return permissioned && !datatableNameTakesRole(name) ? defaultRole : undefined +} + +/** `datatable://`, with `?role=` when a role is named. Throws rather than build a + * reference the executor would refuse, or one that would silently mean another role. */ +export function datatableReference(name: string, role: string | undefined): string { + if (role === undefined) return `datatable://${name}` + if (!isDatatableRoleName(role)) { + throw new Error( + `Invalid data table role '${role}': only letters, digits, '_' and '-' are allowed` + ) + } + if (!datatableNameTakesRole(name)) { + throw new Error( + `Data table '${name}' has a '?' in its name, so it can only be used here as its default role. Rename it to connect as role '${role}'.` + ) + } + return `datatable://${name}?role=${role}` +} + +export type DatatableRowAction = 'migrations' | 'roles' | 'export' | 'import' diff --git a/frontend/src/lib/components/dbdiagram/DbSchemaDiagram.svelte b/frontend/src/lib/components/dbdiagram/DbSchemaDiagram.svelte new file mode 100644 index 0000000000..d9b2752374 --- /dev/null +++ b/frontend/src/lib/components/dbdiagram/DbSchemaDiagram.svelte @@ -0,0 +1,171 @@ + + +
+ {#if error} +
+ {error} +
+ {/if} + {#if tables.length === 0} +
+ Check tables in the sidebar to draw them here +
+ {:else} + + highlight.clear()} + > +
+ + + + + + + {/if} + {#if loading} +
+ + Loading relationships +
+ {/if} +
+ + diff --git a/frontend/src/lib/components/dbdiagram/DbTableNode.svelte b/frontend/src/lib/components/dbdiagram/DbTableNode.svelte new file mode 100644 index 0000000000..d17faaa006 --- /dev/null +++ b/frontend/src/lib/components/dbdiagram/DbTableNode.svelte @@ -0,0 +1,118 @@ + + + +
highlight.hover({ table: table.key })} + onmouseleave={() => highlight.hover(undefined)} + onclick={() => highlight.togglePin({ table: table.key })} + role="button" + tabindex="-1" +> + + +
+ + {table.table} + {table.schema} +
+ + {#each columns as column (column.name)} + {@const columnKey = columnKeyIn(table.key, column.name)} + {@const isLit = highlight.columns?.has(columnKey)} + +
highlight.hover({ table: table.key, column: column.name })} + onmouseleave={() => highlight.hover({ table: table.key })} + onclick={(e) => { + e.stopPropagation() + highlight.togglePin({ table: table.key, column: column.name }) + }} + > + {#if column.isPrimaryKey} + + {:else if data.index.referencingColumns.has(columnKey)} + + {:else} + + {/if} + {column.name} + {column.datatype} +
+ {/each} + + {#if hasExpandToggle(table)} +
+ +
+ {/if} +
diff --git a/frontend/src/lib/components/dbdiagram/DiagramAutoFit.svelte b/frontend/src/lib/components/dbdiagram/DiagramAutoFit.svelte new file mode 100644 index 0000000000..b271886abd --- /dev/null +++ b/frontend/src/lib/components/dbdiagram/DiagramAutoFit.svelte @@ -0,0 +1,20 @@ + diff --git a/frontend/src/lib/components/dbdiagram/dbDiagramHighlight.svelte.ts b/frontend/src/lib/components/dbdiagram/dbDiagramHighlight.svelte.ts new file mode 100644 index 0000000000..54f52b85ca --- /dev/null +++ b/frontend/src/lib/components/dbdiagram/dbDiagramHighlight.svelte.ts @@ -0,0 +1,80 @@ +import { getContext, setContext } from 'svelte' +import { columnKeyIn, type RelationIndex } from '../dbRelations' + +/** What the pointer is on: a whole table, or one of its columns. */ +export type HighlightTarget = { table: string; column?: string } + +const EMPTY: ReadonlySet = new Set() + +/** + * The diagram draws no edges, so the relationships only exist while something is + * highlighted. This owns that: what is hovered (or pinned by a click), and which + * tables and columns it lights up. + */ +export class DbDiagramHighlight { + #getIndex: () => RelationIndex + + hovered = $state(undefined) + /** Set by clicking, so the relations can be read without holding the pointer + * still. A hover takes precedence while it lasts. */ + pinned = $state(undefined) + + target = $derived(this.hovered ?? this.pinned) + + /** Tables that stay lit. Undefined means nothing is highlighted and every + * table renders at full strength. */ + tables = $derived.by | undefined>(() => { + const target = this.target + if (!target) return undefined + if (target.column) { + const columnTable = this.#getIndex().columnTable + const tables = new Set([target.table]) + for (const column of this.columns ?? EMPTY) { + const table = columnTable.get(column) + if (table) tables.add(table) + } + return tables + } + return new Set([target.table, ...(this.#getIndex().relatedTables.get(target.table) ?? EMPTY)]) + }) + + /** Columns that stay lit inside the highlighted tables. */ + columns = $derived.by | undefined>(() => { + const target = this.target + if (!target) return undefined + const index = this.#getIndex() + if (target.column) { + const key = columnKeyIn(target.table, target.column) + return new Set([key, ...(index.linkedColumns.get(key) ?? EMPTY)]) + } + return index.relatedColumns.get(target.table) ?? EMPTY + }) + + constructor(getIndex: () => RelationIndex) { + this.#getIndex = getIndex + } + + hover(target: HighlightTarget | undefined) { + this.hovered = target + } + + /** Clicking the already-pinned target unpins it. */ + togglePin(target: HighlightTarget) { + const current = this.pinned + this.pinned = + current && current.table === target.table && current.column === target.column + ? undefined + : target + } + + clear() { + this.hovered = undefined + this.pinned = undefined + } +} + +const contextKey = 'DbDiagramHighlight' + +export const setDbDiagramHighlight = (highlight: DbDiagramHighlight) => + setContext(contextKey, highlight) +export const getDbDiagramHighlight = () => getContext(contextKey) diff --git a/frontend/src/lib/components/dbdiagram/dbDiagramModel.test.ts b/frontend/src/lib/components/dbdiagram/dbDiagramModel.test.ts new file mode 100644 index 0000000000..8b6b584860 --- /dev/null +++ b/frontend/src/lib/components/dbdiagram/dbDiagramModel.test.ts @@ -0,0 +1,45 @@ +import { describe, expect, it } from 'vitest' +import { CARD_WIDTH, cardHeight, layoutTables, type DiagramTable } from './dbDiagramModel' + +function table(name: string, columnCount: number): DiagramTable { + return { + key: `shop.${name}`, + schema: 'shop', + table: name, + columns: Array.from({ length: columnCount }, (_, i) => ({ + name: `c${i}`, + datatype: 'text', + isPrimaryKey: i === 0 + })) + } +} + +describe('layoutTables', () => { + // Runs again on every checkbox click, so cards must not land on top of each + // other for any mix of card heights. + it('never overlaps two cards', () => { + const tables = Array.from({ length: 40 }, (_, i) => table(`t${i}`, 1 + (i % 17))) + const expanded = new Set(['shop.t3', 'shop.t20']) + const positions = layoutTables(tables, expanded) + + const boxes = tables.map((t) => ({ + x: positions[t.key].x, + y: positions[t.key].y, + h: cardHeight(t, expanded.has(t.key)) + })) + for (let i = 0; i < boxes.length; i++) { + for (let j = i + 1; j < boxes.length; j++) { + const a = boxes[i] + const b = boxes[j] + const apart = + a.x + CARD_WIDTH <= b.x || b.x + CARD_WIDTH <= a.x || a.y + a.h <= b.y || b.y + b.h <= a.y + expect(apart, `${tables[i].key} overlaps ${tables[j].key}`).toBe(true) + } + } + }) + + it('places the same tables in the same spots every time', () => { + const tables = Array.from({ length: 12 }, (_, i) => table(`t${i}`, 3 + i)) + expect(layoutTables(tables, new Set())).toEqual(layoutTables(tables, new Set())) + }) +}) diff --git a/frontend/src/lib/components/dbdiagram/dbDiagramModel.ts b/frontend/src/lib/components/dbdiagram/dbDiagramModel.ts new file mode 100644 index 0000000000..ccd32cdb22 --- /dev/null +++ b/frontend/src/lib/components/dbdiagram/dbDiagramModel.ts @@ -0,0 +1,131 @@ +import type { DBSchema } from '$lib/stores' +import type { ColumnDef } from '../apps/components/display/dbtable/utils' +import type { SelectedTable } from '../DBManager.svelte' +import { tableKeyOf } from '../dbRelations' + +export type DiagramColumn = { + name: string + datatype: string + isPrimaryKey: boolean +} + +export type DiagramTable = { + /** `schema.table`, the key relations and highlights are addressed by. */ + key: string + schema: string + table: string + columns: DiagramColumn[] +} + +/** Card geometry. The node component must render exactly the rows the layout + * budgeted for, or cards overlap. */ +export const CARD_WIDTH = 248 +export const HEADER_HEIGHT = 34 +export const ROW_HEIGHT = 22 +const CARD_PADDING = 6 +const GAP_X = 32 +const GAP_Y = 20 +/** Rows shown before a card collapses into a "+N more" row. */ +export const COLLAPSED_ROW_LIMIT = 12 +/** Cards are packed towards this width/height ratio, so the canvas opens on + * something screen-shaped rather than one very long column. */ +const TARGET_ASPECT = 1.7 + +export function visibleColumns(table: DiagramTable, expanded: boolean): DiagramColumn[] { + return expanded ? table.columns : table.columns.slice(0, COLLAPSED_ROW_LIMIT) +} + +export function hiddenColumnCount(table: DiagramTable, expanded: boolean): number { + return table.columns.length - visibleColumns(table, expanded).length +} + +/** Whether the card carries a row to expand or collapse its column list. */ +export function hasExpandToggle(table: DiagramTable): boolean { + return table.columns.length > COLLAPSED_ROW_LIMIT +} + +export function cardHeight(table: DiagramTable, expanded: boolean): number { + const rows = visibleColumns(table, expanded).length + (hasExpandToggle(table) ? 1 : 0) + return HEADER_HEIGHT + rows * ROW_HEIGHT + CARD_PADDING +} + +/** The tables to draw, in the order they are laid out. */ +export function buildDiagramTables( + dbSchema: DBSchema, + colDefs: Record | undefined, + selected: SelectedTable[] +): DiagramTable[] { + if (dbSchema.lang === 'graphql') return [] + const tables: DiagramTable[] = [] + + for (const { schema, table } of selected) { + const key = tableKeyOf({ schema, table }) + const fromColDefs = colDefs?.[key] + let columns: DiagramColumn[] + if (fromColDefs?.length) { + columns = fromColDefs.map((col) => ({ + name: col.field ?? '', + datatype: col.datatype ?? '', + isPrimaryKey: !!col.isprimarykey + })) + } else { + // The cached schema has no primary-key flag; it only stands in until the + // column metadata query lands. + const schemaColumns = dbSchema.schema[schema]?.[table] + if (!schemaColumns) continue + columns = Object.entries(schemaColumns).map(([name, col]) => ({ + name, + datatype: col.type, + isPrimaryKey: false + })) + } + tables.push({ key, schema, table, columns }) + } + + tables.sort((a, b) => a.schema.localeCompare(b.schema) || a.table.localeCompare(b.table)) + return tables +} + +/** Packs the cards into balanced columns, in reading order. Runs again on every + * change to the selection, so it must be deterministic: the same tables always + * land in the same place. */ +export function layoutTables( + tables: DiagramTable[], + expanded: Set +): Record { + const heights = tables.map((t) => cardHeight(t, expanded.has(t.key))) + const stacked = heights.reduce((a, b) => a + b, 0) + GAP_Y * Math.max(0, tables.length - 1) + + let columnCount = 1 + let bestScore = Infinity + for (let c = 1; c <= tables.length; c++) { + const width = c * CARD_WIDTH + (c - 1) * GAP_X + const height = stacked / c + // Compared in log space so twice-too-wide and twice-too-tall score alike. + const score = Math.abs(Math.log(width / height / TARGET_ASPECT)) + if (score < bestScore) { + bestScore = score + columnCount = c + } + } + + const target = stacked / columnCount + const positions: Record = {} + let column = 0 + let columnHeight = 0 + + tables.forEach((table, i) => { + const height = heights[i] + // Overshooting the target by more than stopping short of it costs less in + // the next column. + const overshoot = columnHeight + height - target + if (column < columnCount - 1 && columnHeight > 0 && overshoot > target - columnHeight) { + column++ + columnHeight = 0 + } + positions[table.key] = { x: column * (CARD_WIDTH + GAP_X), y: columnHeight } + columnHeight += height + GAP_Y + }) + + return positions +} diff --git a/frontend/src/lib/components/dbt/DbtEditor.svelte b/frontend/src/lib/components/dbt/DbtEditor.svelte index 4b30f8fa28..51a029c05a 100644 --- a/frontend/src/lib/components/dbt/DbtEditor.svelte +++ b/frontend/src/lib/components/dbt/DbtEditor.svelte @@ -11,7 +11,6 @@ import { createEventDispatcher, onDestroy, onMount } from 'svelte' import type { Schema, SupportedLanguage } from '$lib/common' import type { Preview, ScriptModule } from '$lib/gen' - import { workspaceStore } from '$lib/stores' import { emptySchema } from '$lib/utils' import { inferArgs } from '$lib/infer' import { Pane, Splitpanes } from 'svelte-splitpanes' @@ -54,6 +53,9 @@ import { ChevronDown, CornerDownLeft, Play, Plus } from 'lucide-svelte' import type { ScriptEditorWhitelabelCustomUi } from '../custom_ui' import { processSecretArgs } from '../secretArgUtils' + import { useOperatingWorkspace } from '$lib/components/operatingWorkspace.svelte' + + const operatingWorkspace = useOperatingWorkspace() let { schema = $bindable(), @@ -86,7 +88,7 @@ } = $props() const dispatch = createEventDispatcher() - let opWs = $derived(workspaceOverride ?? $workspaceStore) + let opWs = $derived(workspaceOverride ?? $operatingWorkspace) /** The open file, or `null` for the descriptor at the project root. */ let openFile = $state(null) diff --git a/frontend/src/lib/components/details/ErrorHandlerToggleButtonV2.svelte b/frontend/src/lib/components/details/ErrorHandlerToggleButtonV2.svelte index 51cb389616..2aec483e32 100644 --- a/frontend/src/lib/components/details/ErrorHandlerToggleButtonV2.svelte +++ b/frontend/src/lib/components/details/ErrorHandlerToggleButtonV2.svelte @@ -3,7 +3,9 @@ import { FlowService, ScriptService } from '$lib/gen' import { sendUserToast } from '$lib/toast' - import { workspaceStore } from '$lib/stores' + import { useOperatingWorkspace } from '$lib/components/operatingWorkspace.svelte' + + const operatingWorkspace = useOperatingWorkspace() interface Props { kind: 'script' | 'flow' @@ -25,11 +27,11 @@ async function toggleErrorHandler(): Promise { toggleState = !toggleState - if ($workspaceStore !== undefined) { + if ($operatingWorkspace !== undefined) { try { if (kind === 'flow') { await FlowService.toggleWorkspaceErrorHandlerForFlow({ - workspace: $workspaceStore, + workspace: $operatingWorkspace, path: scriptOrFlowPath, requestBody: { muted: !errorHandlerMuted @@ -37,7 +39,7 @@ }) } else { await ScriptService.toggleWorkspaceErrorHandlerForScript({ - workspace: $workspaceStore, + workspace: $operatingWorkspace, path: scriptOrFlowPath, requestBody: { muted: !errorHandlerMuted diff --git a/frontend/src/lib/components/diff_drawer.ts b/frontend/src/lib/components/diff_drawer.ts index c6e9b5cdf8..c1e7e8980a 100644 --- a/frontend/src/lib/components/diff_drawer.ts +++ b/frontend/src/lib/components/diff_drawer.ts @@ -1,9 +1,53 @@ import type { Value } from '$lib/utils' +export type DiffVersionOption = { + /** Opaque to the drawer — a script hash, a flow version id, an app version id. */ + id: string + /** Identifies the version: its number and hash. Carries the weight in the list. */ + label: string + /** Who deployed it, and when — rendered under the label in secondary text, so the + * version reads first and the attribution second. */ + subtitle?: string + /** The version currently deployed. */ + isHead?: boolean +} + +/** How many deployed versions the diff picker asks for at a time. The history endpoints + * still answer whole when nobody asks — the panels and the CLI read them that way — so + * this is the picker's own appetite, not their default. */ +export const VERSION_PAGE_SIZE = 20 + export type DiffDrawerDiff = | { mode: 'normal' deployed: Value + /** Names the deployed side, e.g. `Deployed d2154d55 by bob`. The version + * identity is stripped from the displayed metadata, so without this the + * reader cannot tell what their draft is being compared against. */ + deployedLabel?: string + /** Deployed versions the reader can compare against, newest first. The + * drawer only renders the picker; each editor supplies its own list and + * fetcher because scripts, flows and apps identify versions differently. */ + versions?: DiffVersionOption[] + /** Loads one version's payload. Returning `undefined` leaves the current + * comparison in place rather than blanking the diff. */ + loadVersion?: (id: string) => Promise + /** Fetches the next, older page of `versions`. `versions` holds one page so the + * drawer opens without waiting on a path a pipeline has deployed thousands of + * times; the reader asks for the rest. Returning nothing ends the list. */ + loadMoreVersions?: () => Promise + /** Moves the draft's base to the head and keeps its content, rendered as a + * header action so the user takes the latest with the diff in front of them. + * Called with the version the drawer is showing as head (from `versions`), so + * the base adopted is the one the reader just looked at. */ + onTakeLatest?: (head?: string) => void | Promise + /** The version the draft forked from. The drawer offers `onTakeLatest` only + * while it differs from the head, and passes that head to it. */ + draftBase?: string + /** The version the deployed payload passed here came from, used as the head when + * no `versions` list loaded. Without either, the drawer cannot tell whether the + * draft is behind and offers nothing. */ + deployedHead?: string draft?: Value | undefined current: Value defaultDiffType?: 'deployed' | 'draft' @@ -18,7 +62,19 @@ export type DiffDrawerDiff = } export interface DiffDrawerI { - openDrawer: () => void + /** Pass the token from `beginOpening` to continue that opening; called without one, + * the drawer claims a fresh opening, so any reuse invalidates a fetch still in + * flight rather than being overwritten by it. */ + openDrawer: (token?: number) => void closeDrawer: () => void - setDiff: (diff: DiffDrawerDiff) => void + /** Takes the opening's token on the same terms as `openDrawer`. */ + setDiff: (diff: DiffDrawerDiff, token?: number) => void + /** Claim the drawer for one opening. Filling it takes awaited fetches, and a path + * change remounts the editor while this drawer stays, so the token lives here: an + * editor checks `ownsOpening` before every write and drops the opening it started + * when it goes away. */ + beginOpening: () => number + ownsOpening: (token: number) => boolean + /** Drop that opening and, when it still owns the drawer, what it put on screen. */ + abandonOpening: (token: number) => void } diff --git a/frontend/src/lib/components/flow_builder.ts b/frontend/src/lib/components/flow_builder.ts index ce848c6f83..f763e92faa 100644 --- a/frontend/src/lib/components/flow_builder.ts +++ b/frontend/src/lib/components/flow_builder.ts @@ -10,6 +10,14 @@ import type { WorkspaceItem } from './workspacePicker' export type FlowBuilderProps = { initialPath?: string + /** The draft row's own path (the URL path). `initialPath` is re-seeded from the + * draft's `draft_path` so the topbar shows the pending name, so it can't be used + * to resolve what is actually deployed. */ + userDraftPath?: string + /** Moves the draft's base to the head and keeps its content; offered in the diff + * drawer while the draft is behind, and called with the version the drawer shows + * as head. */ + onTakeLatest?: (head?: string) => void | Promise pathStoreInit?: string | undefined newFlow: boolean selectedId: string | undefined @@ -24,9 +32,10 @@ export type FlowBuilderProps = { disabledFlowInputs?: boolean savedPrimarySchedule?: ScheduleTrigger | undefined // used to set the primary schedule in the legacy primaryScheduleStore version?: number | undefined - /** flow_version the draft was forked from; when set, the deploy-time staleness - * check compares it (not the load-time head `version`) against the latest. */ - draftBaseVersion?: number | undefined + /** flow_version the draft was forked from (as text, like `draft_base`); when + * set, the deploy-time staleness check compares it (not the load-time head + * `version`) against the latest. */ + draftBaseVersion?: string | undefined draftTriggersFromUrl?: Trigger[] | undefined selectedTriggerIndexFromUrl?: number | undefined children?: import('svelte').Snippet diff --git a/frontend/src/lib/components/flows/FlowAssetsHandler.svelte b/frontend/src/lib/components/flows/FlowAssetsHandler.svelte index 947eca1a67..86fff85437 100644 --- a/frontend/src/lib/components/flows/FlowAssetsHandler.svelte +++ b/frontend/src/lib/components/flows/FlowAssetsHandler.svelte @@ -43,11 +43,13 @@ type RawScript } from '$lib/gen' import { deepEqual } from 'fast-equals' - import { workspaceStore } from '$lib/stores' import S3FilePicker from '../S3FilePicker.svelte' import ResourceEditorDrawer from '../ResourceEditorDrawer.svelte' import { watch } from 'runed' import { sendUserToast } from '$lib/toast' + import { useOperatingWorkspace } from '$lib/components/operatingWorkspace.svelte' + + const operatingWorkspace = useOperatingWorkspace() let { modules, @@ -63,7 +65,7 @@ const flowGraphAssetsCtx = getContext('FlowGraphAssetContext') const { selectionManager, opWorkspace } = getContext('FlowEditorContext') || {} - let opWs = $derived(opWorkspace?.() ?? $workspaceStore) + let opWs = $derived(opWorkspace?.() ?? $operatingWorkspace) // Expose the acting workspace to the asset explore controls (ExploreAssetButton // reads it from this context; the DB manager / S3 picker act on it). $effect(() => { diff --git a/frontend/src/lib/components/flows/FlowEditor.svelte b/frontend/src/lib/components/flows/FlowEditor.svelte index bf25a1dcda..a9c3a40e8a 100644 --- a/frontend/src/lib/components/flows/FlowEditor.svelte +++ b/frontend/src/lib/components/flows/FlowEditor.svelte @@ -40,14 +40,21 @@ import FlowPanelPlacementPicker from './common/FlowPanelPlacementPicker.svelte' import { prefersSessionHandoff } from '../copilot/chat/global/gate' import { openSourceInSession } from '$lib/components/sessions/sessionSwitch.svelte' - import { userStore, workspaceStore } from '$lib/stores' + import { + useOperatingUser, + useOperatingWorkspace + } from '$lib/components/operatingWorkspace.svelte' + + const operatingWorkspace = useOperatingWorkspace() + const operatingUser = useOperatingUser() + const actingUser = $derived(operatingUser.current) const { flowStore, selectionManager, pathStore, opWorkspace } = getContext('FlowEditorContext') // Flow paths repeat across workspaces, and a session keeps every tab it has visited alive, so two // editors can hold the same path at once. Both halves are needed to tell them apart. - let editorWorkspace = $derived(opWorkspace?.() ?? $workspaceStore) + let editorWorkspace = $derived(opWorkspace?.() ?? $operatingWorkspace) function targetWorkspace(t: AgentEditorTarget): string | undefined { - return t.workspace ?? $workspaceStore + return t.workspace ?? $operatingWorkspace } const sessionScopedManager = getContext('aiChatManager') @@ -404,7 +411,7 @@ if ( !sessionScopedManager && sessionOpen && - prefersSessionHandoff($userStore?.operator) + prefersSessionHandoff(actingUser?.operator) ) { void openSourceInSession(sessionOpen, { previewParams: { selected: detail.moduleId }, diff --git a/frontend/src/lib/components/flows/FlowHistoryInner.svelte b/frontend/src/lib/components/flows/FlowHistoryInner.svelte index 1f906c1e45..87c2306339 100644 --- a/frontend/src/lib/components/flows/FlowHistoryInner.svelte +++ b/frontend/src/lib/components/flows/FlowHistoryInner.svelte @@ -5,12 +5,14 @@ import { Pane, Splitpanes } from 'svelte-splitpanes' import { classNames, displayDate, emptyString, sendUserToast } from '$lib/utils' import { type Flow, FlowService, type FlowVersion } from '$lib/gen' - import { workspaceStore } from '$lib/stores' import { Skeleton } from '$lib/components/common' import Button from '../common/button/Button.svelte' import { ArrowRight, Loader2, Pencil, X } from 'lucide-svelte' import { getContext } from 'svelte' import type { FlowEditorContext } from './types' + import { useOperatingWorkspace } from '$lib/components/operatingWorkspace.svelte' + + const operatingWorkspace = useOperatingWorkspace() interface Props { path: string @@ -21,7 +23,7 @@ let { path, allowFork = false, onHistoryRestore }: Props = $props() const flowEditorContext = getContext('FlowEditorContext') - let opWs = $derived(flowEditorContext?.opWorkspace?.() ?? $workspaceStore) + let opWs = $derived(flowEditorContext?.opWorkspace?.() ?? $operatingWorkspace) let loading: boolean = $state(false) diff --git a/frontend/src/lib/components/flows/common/FlowCardHeader.svelte b/frontend/src/lib/components/flows/common/FlowCardHeader.svelte index 063367cc85..5776c88025 100644 --- a/frontend/src/lib/components/flows/common/FlowCardHeader.svelte +++ b/frontend/src/lib/components/flows/common/FlowCardHeader.svelte @@ -19,7 +19,7 @@ import FlowPanelChrome from './FlowPanelChrome.svelte' import type { FlowBuilderWhitelabelCustomUi } from '$lib/components/custom_ui' import DropdownV2 from '$lib/components/DropdownV2.svelte' - import { hubBaseUrlStore, workspaceStore } from '$lib/stores' + import { hubBaseUrlStore } from '$lib/stores' import { DEFAULT_HUB_BASE_URL, PRIVATE_HUB_MIN_VERSION } from '$lib/hub' import { getLatestHashForScript } from '$lib/scripts' import { sendUserToast, type Item } from '$lib/utils' @@ -27,6 +27,9 @@ import { getToolNameError } from '$lib/components/flows/agentToolUtils' import { logFeatureUsage } from '$lib/utils/featureUsage' import autosize from '$lib/autosize' + import { useOperatingWorkspace } from '$lib/components/operatingWorkspace.svelte' + + const operatingWorkspace = useOperatingWorkspace() interface Props { flowModuleValue?: FlowModuleValue | undefined @@ -65,7 +68,7 @@ const flowEditorContext = getContext('FlowEditorContext') const { scriptEditorDrawer, workspaceScriptSettingsDrawer } = flowEditorContext - let opWs = $derived(flowEditorContext?.opWorkspace?.() ?? $workspaceStore) + let opWs = $derived(flowEditorContext?.opWorkspace?.() ?? $operatingWorkspace) const scriptPath = $derived(flowModuleValue?.type === 'script' ? flowModuleValue.path : undefined) const pinnedHash = $derived(flowModuleValue?.type === 'script' ? flowModuleValue.hash : undefined) const isHub = $derived(scriptPath?.startsWith('hub/') ?? false) diff --git a/frontend/src/lib/components/flows/content/AgentEditorModal.svelte b/frontend/src/lib/components/flows/content/AgentEditorModal.svelte index 11e59769fc..a6e113f39b 100644 --- a/frontend/src/lib/components/flows/content/AgentEditorModal.svelte +++ b/frontend/src/lib/components/flows/content/AgentEditorModal.svelte @@ -12,7 +12,6 @@ import EvalsPane from '$lib/components/aiEvals/EvalsPane.svelte' import type { EvalsLocation } from '$lib/components/aiEvals/evalUtils' import { ResourceService, type AgentDraft } from '$lib/gen' - import { workspaceStore } from '$lib/stores' import { agentEditorTarget, agentWriteCount, @@ -27,6 +26,9 @@ import { publishLinkedAgentTools } from '../flowState' import { linkedModulesForAgent, linkedToolsScope } from '../linkedAgentToolsStore.svelte' import AgentEditorHost from './AgentEditorHost.svelte' + import { useOperatingWorkspace } from '$lib/components/operatingWorkspace.svelte' + + const operatingWorkspace = useOperatingWorkspace() interface Props { enableAi?: boolean @@ -61,7 +63,7 @@ if (p) untrack(() => (lastShownAgent = p)) }) - let ws = $derived(target?.workspace ?? $workspaceStore) + let ws = $derived(target?.workspace ?? $operatingWorkspace) let host = $state | undefined>(undefined) let versionDrawer: Drawer | undefined = $state(undefined) let saving = $state(false) diff --git a/frontend/src/lib/components/flows/content/AgentResourceBar.svelte b/frontend/src/lib/components/flows/content/AgentResourceBar.svelte index 9bfcb222b2..83be7c876e 100644 --- a/frontend/src/lib/components/flows/content/AgentResourceBar.svelte +++ b/frontend/src/lib/components/flows/content/AgentResourceBar.svelte @@ -7,7 +7,6 @@ import Label from '$lib/components/Label.svelte' import ResourcePathHint from '$lib/components/ResourcePathHint.svelte' import { ResourceService, type InputTransform, type Resource } from '$lib/gen' - import { workspaceStore } from '$lib/stores' import { sendUserToast } from '$lib/toast' import { Bot, ChevronDown, ChevronUp, Save, Unlink, Pencil } from 'lucide-svelte' import { @@ -46,6 +45,9 @@ import type { AgentTool as AgentToolStrict } from '../agentToolUtils' import { resource } from 'runed' import { untrack } from 'svelte' + import { useOperatingWorkspace } from '$lib/components/operatingWorkspace.svelte' + + const operatingWorkspace = useOperatingWorkspace() let { agent = $bindable(), @@ -65,7 +67,7 @@ toolInputs: Record> moduleId: string // The workspace the flow editor operates on (differs from the nav workspace in session/fork - // editors). All resource reads/writes must target it, not $workspaceStore. + // editors). All resource reads/writes must target it, not the navigation workspace. opWorkspace?: string // Scope for the linked-agent tools store (the flow path); must match what the graph reads. flowPath?: string @@ -80,7 +82,7 @@ linkedMemory?: { memory: unknown } | undefined } = $props() - let ws = $derived(opWorkspace ?? $workspaceStore) + let ws = $derived(opWorkspace ?? $operatingWorkspace) // How many times the linked agent has been written, from anywhere: this card's own save, or a // deploy from the agent editor mounted alongside it. Both reads below key on it, so neither diff --git a/frontend/src/lib/components/flows/content/AiAgentStepInputs.svelte b/frontend/src/lib/components/flows/content/AiAgentStepInputs.svelte index 37fcfe8d1c..33a15922f5 100644 --- a/frontend/src/lib/components/flows/content/AiAgentStepInputs.svelte +++ b/frontend/src/lib/components/flows/content/AiAgentStepInputs.svelte @@ -30,7 +30,6 @@ import type { Schema } from '$lib/common' import { deepEqual } from 'fast-equals' import { type InputTransform } from '$lib/gen' - import { workspaceStore } from '$lib/stores' import { allTrue, type DynamicInput as DynamicInputTypes } from '$lib/utils' import { getContext, untrack, type Snippet } from 'svelte' import { SvelteSet } from 'svelte/reactivity' @@ -69,6 +68,9 @@ import AgentToolRoster from './AgentToolRoster.svelte' import AgentMemoryNotes from './AgentMemoryNotes.svelte' import { memoryOptionLabel, memoryPropertyFor } from '../flowInfers' + import { useOperatingWorkspace } from '$lib/components/operatingWorkspace.svelte' + + const operatingWorkspace = useOperatingWorkspace() interface Props { schema: Schema | { properties?: Record } @@ -148,7 +150,7 @@ linkedMemory = undefined }: Props = $props() - let ws = $derived(workspace ?? $workspaceStore) + let ws = $derived(workspace ?? $operatingWorkspace) let inputCheck: { [id: string]: boolean } = $state({}) diff --git a/frontend/src/lib/components/flows/content/ExpandedSubflowStep.svelte b/frontend/src/lib/components/flows/content/ExpandedSubflowStep.svelte index 4aede5e3d1..979717dae1 100644 --- a/frontend/src/lib/components/flows/content/ExpandedSubflowStep.svelte +++ b/frontend/src/lib/components/flows/content/ExpandedSubflowStep.svelte @@ -1,7 +1,6 @@ -
- - {selection.size} selected - -
- - - {#if targets('archive').length > 0 || targets('unarchive').length === 0} + + {items.length} selected + +
- {/if} - {#if targets('unarchive').length > 0} - - {/if} - open('discard'), - disabled: targets('discard').length === 0, - // Only while blocked: a disabled entry can't open its modal, so the reason - // has to live here — and an enabled one would render a pointless ⓘ. - tooltip: targets('discard').length === 0 ? actionTitle('discard') : undefined - }, - { - displayName: `Delete${countSuffix('delete')}`, - icon: Trash, - type: 'delete' as const, - action: () => open('delete'), - disabled: targets('delete').length === 0, - tooltip: targets('delete').length === 0 ? actionTitle('delete') : undefined - } - ]} - > - {#snippet buttonReplacement()} + + {#if targets('archive').length > 0 || targets('unarchive').length === 0} - {/snippet} - -
- + {/if} + open('discard'), + disabled: targets('discard').length === 0, + // Only while blocked: a disabled entry can't open its modal, so the reason + // has to live here — and an enabled one would render a pointless ⓘ. + tooltip: targets('discard').length === 0 ? actionTitle('discard') : undefined + }, + { + displayName: `Delete${countSuffix('delete')}`, + icon: Trash, + type: 'delete' as const, + action: () => open('delete'), + disabled: targets('delete').length === 0, + tooltip: targets('delete').length === 0 ? actionTitle('delete') : undefined + } + ]} + > + {#snippet buttonReplacement()} + + {/snippet} + +
+
`${i.path} → ${movedPath(i, target)}`) + pendingItems.map((i) => `${sourcePath(i)} → ${movedPath(i, target)}`) )} {/if} {:else if pending === 'discard'} diff --git a/frontend/src/lib/components/home/ItemsList.svelte b/frontend/src/lib/components/home/ItemsList.svelte index 7038a316fa..f86ec9d5bf 100644 --- a/frontend/src/lib/components/home/ItemsList.svelte +++ b/frontend/src/lib/components/home/ItemsList.svelte @@ -20,7 +20,6 @@ import type uFuzzy from '@leeoniya/ufuzzy' import { ArrowDownUp, - CheckSquare, ChevronsDownUp, ChevronsUpDown, Code2, @@ -1805,20 +1804,10 @@ {/if} {#if !loading && !contentActive && !workspaceEmpty} - +
- {#if homeSelection.available && !homeSelection.active} - {:else} - download + download {/if}
diff --git a/frontend/src/lib/components/raw_apps/RawAppDataTableDrawer.svelte b/frontend/src/lib/components/raw_apps/RawAppDataTableDrawer.svelte index 5f0488638d..7ccf89a275 100644 --- a/frontend/src/lib/components/raw_apps/RawAppDataTableDrawer.svelte +++ b/frontend/src/lib/components/raw_apps/RawAppDataTableDrawer.svelte @@ -1,35 +1,50 @@ + +{#snippet loading()} +
+ +
+{/snippet} + +
+ {#if eeLocked} +
This trigger requires an enterprise license.
+ {:else if lookup.loading || !lookup.current} + {@render loading()} + {:else if !lookup.current.exists} +
+

+ {pageItemKindLabel(item)} {item.path} no longer exists in this + workspace. +

+ +
+ {:else} + {#key `${item.kind}:${triggerKey}:${item.path}:${workspaceId}:${reloadNonce}:${savedNonce}`} + {#if item.kind === 'variable'} + + {:else if item.kind === 'resource'} + { + if (path !== undefined) onSaved(path) + }} + onRestored={() => savedNonce++} + /> + {:else if triggerKey} + {#await TRIGGER_EDITORS[triggerKey]()} + {@render loading()} + {:then Module} + onSaved(path)} + /> + {/await} + {/if} + {/key} + {/if} +
diff --git a/frontend/src/lib/components/sessions/PreviewTabHost.svelte b/frontend/src/lib/components/sessions/PreviewTabHost.svelte index 3dee7dd1b3..b88907bf02 100644 --- a/frontend/src/lib/components/sessions/PreviewTabHost.svelte +++ b/frontend/src/lib/components/sessions/PreviewTabHost.svelte @@ -103,12 +103,19 @@ applyPageIframeTheme(darkMode) }) + // A page item's editor reads its draft only when it loads, so a reload remounts it. + let pageItemReloadNonce = $state(0) + export function reload() { // A live editor shares the runtime store the chat mutates, so generic chat // edits are already reflected — no reload needed. Deploys refresh it via // each editor view's onDeploy → runtime.syncPreviewWithDeployed. So only the // iframe fallback (a separate page) has to be told to refresh. if (slot.kind === 'editor') return + if (slot.kind === 'pageitem') { + pageItemReloadNonce++ + return + } try { const win = frame?.contentWindow if (!win) return @@ -311,6 +318,22 @@ {/await} {/if}
+{:else if slot.kind === 'pageitem' && mounted && runtime} +
+ + {#if overlayHostEl} + {#await import('./PageItemEditorView.svelte')} + {@render editorLoading()} + {:then Module} + + {/await} + {/if} +
{:else if slot.kind === 'artifact' && mounted}
c.type !== 'app_dom_selector' || (c.selector === selector && c.appPath === path) ) - if (runtime.manager.loading) { + if (runtime.manager.loading || runtime.manager.sendInFlight || runtime.manager.sendPending) { runtime.manager.queueMessage(prompt, [], snapshot) } else { void runtime.manager.sendRequest({ instructions: prompt, contextOverride: snapshot }) @@ -240,6 +241,9 @@ cell.saved.val), reactive state owned by the SessionRuntime class (via rawAppCell), not by a component ancestor — so Svelte's ownership check flags a false positive here. --> + workspaceId) // This tab's storage path, for the editor below: several tabs are mounted at // once and only this one knows which item each is open on. diff --git a/frontend/src/lib/components/sessions/SessionPicker.svelte b/frontend/src/lib/components/sessions/SessionPicker.svelte index e5b62da0eb..ce1ad89246 100644 --- a/frontend/src/lib/components/sessions/SessionPicker.svelte +++ b/frontend/src/lib/components/sessions/SessionPicker.svelte @@ -33,6 +33,7 @@ reconcileAfterWorkspaceChange, renameSession, selectSession, + sessionPageHref, sessionLastActivityAt, sessionState, setNewSessionWorkspace, @@ -44,8 +45,9 @@ import { unreadCountFor } from './sessionUnread.svelte' import Toggle from '$lib/components/Toggle.svelte' import { - getOrCreateRuntime, + ensureSessionChatPeek, getRuntime, + getSessionChatPeek, getSessionChatStatus, removeSession, resetSessionPreviewTabs @@ -87,7 +89,7 @@ // displayMessages array vs. the localStorage-backed lastSeen map; // both are reactive so the badge updates without polling. function unreadFor(session: Session): number { - return unreadCountFor(session.id, getRuntime(session.id)) + return unreadCountFor(session.id, getRuntime(session.id), getSessionChatPeek(session.id)) } // Whether the composer for a session holds non-whitespace text. We @@ -378,14 +380,12 @@ } }) - // Eagerly create a runtime per VISIBLE session so the status dot reflects - // the persisted chat (last message, pending confirmation, etc.) without - // requiring the user to open the session first. Sessions outside the - // current workspace scope are left cold to avoid opening IDB connections - // for unrelated work. + // Read each VISIBLE session's stored chat so the status dot and unread count + // reflect it without the user opening the session. A runtime per listed + // session would hold every listed chat in memory for as long as the page lives. $effect(() => { for (const session of visibleSessions) { - getOrCreateRuntime(session) + if (!getRuntime(session.id)) void ensureSessionChatPeek(session) } }) @@ -402,7 +402,7 @@ // so opening one must not change the user's active (navigation) workspace. // Open the dedicated sessions page; its preview panel iframes the // session's view (captured page / editor target). - await goto(`/sessions?session_name=${encodeURIComponent(session.name)}`) + await goto(sessionPageHref(session.id)) if (restoreFocus) { // goto() resets focus to — put it back on the active session button // so subsequent arrow keys keep navigating the list. @@ -609,14 +609,14 @@ // After deleting the open session, land somewhere usable: the newest remaining // session, else a fresh one. The page derives the visible session from the - // `session_name` query, so leaving the URL on a deleted session would fall + // `session` query, so leaving the URL on a deleted session would fall // through to recovery and open a blank one rather than their recent work. async function openReplacementSession() { const next = sessionState.sessions[0] if (next) await activate(next) else { const fresh = createSession() - await goto(`/sessions?session_name=${encodeURIComponent(fresh.name)}`) + await goto(sessionPageHref(fresh.id)) } } @@ -768,7 +768,9 @@ {/if} {#each group.sessions as session (session.id)} {@const runtime = getRuntime(session.id)} - {@const status = runtime ? getSessionChatStatus(runtime) : 'idle'} + {@const status = runtime + ? getSessionChatStatus(runtime) + : (getSessionChatPeek(session.id)?.status ?? 'idle')} {@const isSelected = sessionActive && session.id === sessionState.currentSessionId} {@const unread = unreadFor(session)} @@ -986,7 +988,9 @@ {/snippet} {#snippet sessionRow(session, indented, treeDepth)} {@const runtime = getRuntime(session.id)} - {@const status = runtime ? getSessionChatStatus(runtime) : 'idle'} + {@const status = runtime + ? getSessionChatStatus(runtime) + : (getSessionChatPeek(session.id)?.status ?? 'idle')} {@const isSelected = sessionActive && session.id === sessionState.currentSessionId} {@const isEditing = editingId === session.id} {@const unread = unreadFor(session)} @@ -1294,7 +1298,7 @@

Delete session {pendingDelete?.summary ?? pendingDelete?.name}{pendingDelete?.summary ?? 'Untitled session'}? This cannot be undone.

{#if pendingDeleteForkId} diff --git a/frontend/src/lib/components/sessions/SessionWrapper.svelte b/frontend/src/lib/components/sessions/SessionWrapper.svelte index b56b72c2be..775b1b150f 100644 --- a/frontend/src/lib/components/sessions/SessionWrapper.svelte +++ b/frontend/src/lib/components/sessions/SessionWrapper.svelte @@ -46,6 +46,7 @@ reconcileAfterWorkspaceChange, renameSession, selectSession, + sessionPageHref, sessionState, setSessionArchived, syncWorkspaceTo, @@ -164,10 +165,10 @@ async function resetToNewSession() { const fresh = createSession() selectSession(fresh.id) - // The page derives the visible session from the `session_name` query, not + // The page derives the visible session from the `session` query, not // currentSessionId: navigate so the URL leaves the deleted/archived session, // which would otherwise fall through to recovery or stay on the archived one. - await goto(`/sessions?session_name=${encodeURIComponent(fresh.name)}`) + await goto(sessionPageHref(fresh.id)) } // If the session targets a forked workspace that's still accessible, @@ -536,7 +537,7 @@

Delete session {session?.summary ?? session?.name}{session?.summary ?? 'Untitled session'}? This cannot be undone.

{#if sessionForkId} @@ -568,7 +569,7 @@

Archive session {session?.summary ?? session?.name}{session?.summary ?? 'Untitled session'}? You can restore it later from the archived list.

{#if sessionForkId} diff --git a/frontend/src/lib/components/sessions/appDraftCodec.test.ts b/frontend/src/lib/components/sessions/appDraftCodec.test.ts index ba0d6f5f2d..8a6ac74f86 100644 --- a/frontend/src/lib/components/sessions/appDraftCodec.test.ts +++ b/frontend/src/lib/components/sessions/appDraftCodec.test.ts @@ -78,3 +78,25 @@ describe('appDraftCodec — draft_path round-trip', () => { expect(back.draft_path).toBe('u/admin/renamed') }) }) + +describe('appDraftCodec — fork base round-trip', () => { + it('carries the draft version both ways', () => { + const draft = runtimeRawAppToDraft(runtime({ parent_version: 3 })) + expect(draft.parent_version).toBe(3) + expect(applyDraftToRuntimeRawApp(runtime({ parent_version: 1 }), draft).parent_version).toBe(3) + }) + + it('keeps an unknown base unknown rather than adopting the runtime version', () => { + const dv: RawAppDraft = { + summary: 'app', + files: {}, + runnables: {}, + data: { tables: [] } as any + } + // This result round-trips into the next save: a fallback here would claim the + // content forked from the version the session happens to hold. + expect(applyDraftToRuntimeRawApp(runtime({ parent_version: 2 }), dv).parent_version).toBe( + undefined + ) + }) +}) diff --git a/frontend/src/lib/components/sessions/appDraftCodec.ts b/frontend/src/lib/components/sessions/appDraftCodec.ts index 10752b8d8f..7d9f4b048f 100644 --- a/frontend/src/lib/components/sessions/appDraftCodec.ts +++ b/frontend/src/lib/components/sessions/appDraftCodec.ts @@ -16,6 +16,9 @@ export type RawAppDraft = { // friendly name (they read `value->>'draft_path'`) — and so editing the path // in the editor changes the persisted draft and triggers an autosave. draft_path?: string + // The app_version the draft forked from. The server derives `draft.base` from + // it, which is what tells a draft that is behind the deployed head. + parent_version?: number } // The shape a raw-app cell's store (`RawAppRuntimeValue` in @@ -30,6 +33,7 @@ export type RuntimeRawApp = { policy: any custom_path?: string draft_path?: string + parent_version?: number } // Strip runtime-only metadata (just `path`, the storage key) when persisting @@ -43,7 +47,8 @@ export function runtimeRawAppToDraft(raw: RuntimeRawApp): RawAppDraft { data: raw.data, policy: raw.policy, custom_path: raw.custom_path, - draft_path: raw.draft_path + draft_path: raw.draft_path, + parent_version: raw.parent_version } } @@ -58,6 +63,10 @@ export function applyDraftToRuntimeRawApp(raw: RuntimeRawApp, dv: RawAppDraft): data: dv.data, policy: dv.policy ?? raw.policy, custom_path: dv.custom_path ?? raw.custom_path, - draft_path: dv.draft_path ?? raw.draft_path + draft_path: dv.draft_path ?? raw.draft_path, + // The incoming draft's own fork base, absence included: this round-trips into the + // next save, so falling back to the runtime's version would give that content a + // base it never forked from and hide that it is behind. + parent_version: dv.parent_version } } diff --git a/frontend/src/lib/components/sessions/pageDrawerSession.ts b/frontend/src/lib/components/sessions/pageDrawerSession.ts index ca7379be20..16d8d9204a 100644 --- a/frontend/src/lib/components/sessions/pageDrawerSession.ts +++ b/frontend/src/lib/components/sessions/pageDrawerSession.ts @@ -11,6 +11,7 @@ import type { UserDraftItemKind } from '$lib/gen' // flow editors, where pulling the filter schemas that module reads views from would make // every trigger's save utils eager. import { + drawerHashFor, pageHref, stripBase, TRIGGER_PAGES, @@ -20,6 +21,7 @@ import { type TriggerKind } from './previewPaths' import type { OpenInSessionSource } from './OpenInSessionButton.svelte' +import { isSessionPreviewFrame } from './sessionMode.svelte' // The draft each page's drawer edits. The preview loads the page in its own // document and reads the draft back from the server, so opening a session has to @@ -63,11 +65,6 @@ async function flushOrRefuse(query: Parameters[0 } } -// How each page addresses a row in its hash. Resources route theirs through an extra -// segment; every other page names the path directly. -const drawerHashFor = (pagePath: string, itemPath: string) => - pagePath === RESOURCES_PATH ? `/resource/${itemPath}` : itemPath - /** * Deep-link the row whose drawer just opened, so the location says what is on screen — a * drawer opened from a row's Edit button is as open as one reached by link, and the chat @@ -87,6 +84,34 @@ export function setPageDrawerAnchor(pagePath: string, itemPath: string | undefin history.replaceState(history.state, '', `${pathname}${search}${anchor}`) } +/** + * Inside a session preview frame, hand a list page row up to the session, which edits it in + * a tab of its own. True when handed off: the caller must then not open its drawer. False + * off that page, and outside a preview frame, where the drawer is how the row is edited. + */ +export function handOffPageDrawer(pagePath: string, itemPath: string | undefined): boolean { + if (!itemPath || !isSessionPreviewFrame()) return false + if (stripBase(window.location.pathname) !== pagePath) return false + try { + window.parent.postMessage( + { type: 'wm.session.openPageItem', pagePath, path: itemPath }, + window.location.origin + ) + } catch { + return false + } + // A frame left on a row's hash claims a row nobody has open here, and reopens its tab on + // every reload. Not through the router: these pages open their drawer from the hash. Once + // more after the event: a row link's `href="#"` lands after its click handler. + const dropAnchor = () => { + const { pathname, search, hash } = window.location + if (hash) history.replaceState(history.state, '', `${pathname}${search}`) + } + dropAnchor() + setTimeout(dropAnchor, 0) + return true +} + /** * Drop the row a list page deep-links, once its drawer closes. The hash is how the row was * requested; leaving it behind makes the location claim a drawer that is no longer open — diff --git a/frontend/src/lib/components/sessions/pageItemLookup.ts b/frontend/src/lib/components/sessions/pageItemLookup.ts new file mode 100644 index 0000000000..bee3fc41c1 --- /dev/null +++ b/frontend/src/lib/components/sessions/pageItemLookup.ts @@ -0,0 +1,62 @@ +import { + AmqpTriggerService, + ApiError, + AzureTriggerService, + EmailTriggerService, + GcpTriggerService, + HttpTriggerService, + KafkaTriggerService, + MqttTriggerService, + NatsTriggerService, + PostgresTriggerService, + ResourceService, + ScheduleService, + SqsTriggerService, + VariableService, + WebsocketTriggerService +} from '$lib/gen' +import type { PageItemRef, TriggerKind } from './previewPaths' + +type Get = (args: { workspace: string; path: string; getDraft: boolean }) => Promise + +const TRIGGER_GETS: Record = { + http: (a) => HttpTriggerService.getHttpTrigger(a), + websocket: (a) => WebsocketTriggerService.getWebsocketTrigger(a), + postgres: (a) => PostgresTriggerService.getPostgresTrigger(a), + kafka: (a) => KafkaTriggerService.getKafkaTrigger(a), + nats: (a) => NatsTriggerService.getNatsTrigger(a), + mqtt: (a) => MqttTriggerService.getMqttTrigger(a), + amqp: (a) => AmqpTriggerService.getAmqpTrigger(a), + sqs: (a) => SqsTriggerService.getSqsTrigger(a), + gcp: (a) => GcpTriggerService.getGcpTrigger(a), + azure: (a) => AzureTriggerService.getAzureTrigger(a), + email: (a) => EmailTriggerService.getEmailTrigger(a) +} + +function getFor(ref: PageItemRef): Get { + switch (ref.kind) { + case 'variable': + return (a) => VariableService.getVariable(a) + case 'resource': + return (a) => ResourceService.getResource(a) + case 'schedule': + return (a) => ScheduleService.getSchedule(a) + case 'trigger': + return TRIGGER_GETS[ref.triggerKind] + } +} + +/** A page item as its editor would load it — deployed, or only a draft — or undefined when it + * is neither: deleted, or a draft that was discarded. Any other failure is thrown, for the + * caller to leave to the editor rather than report as a missing item. */ +export async function lookupPageItem( + ref: PageItemRef, + workspace: string +): Promise | undefined> { + try { + return (await getFor(ref)({ workspace, path: ref.path, getDraft: true })) as Record + } catch (e) { + if (e instanceof ApiError && e.status === 404) return undefined + throw e + } +} diff --git a/frontend/src/lib/components/sessions/previewPaths.ts b/frontend/src/lib/components/sessions/previewPaths.ts index 8d10dc1a32..13d5abca33 100644 --- a/frontend/src/lib/components/sessions/previewPaths.ts +++ b/frontend/src/lib/components/sessions/previewPaths.ts @@ -48,6 +48,87 @@ export const TRIGGER_PAGES: Record + pagePath === RESOURCES_PATH ? `/resource/${itemPath}` : itemPath + +/** The full page a page item is edited on: its list page, with the row's drawer open. */ +export function pageItemPageHref(ref: PageItemRef): string { + const listPath = pageItemListPath(ref) + return `${pageHref(listPath)}#${drawerHashFor(listPath, ref.path)}` +} + +/** The list page a page item is edited from. */ +export function pageItemListPath(ref: PageItemRef): string { + switch (ref.kind) { + case 'variable': + return VARIABLES_PATH + case 'resource': + return RESOURCES_PATH + case 'schedule': + return SCHEDULES_PATH + case 'trigger': + return TRIGGER_PAGES[ref.triggerKind].path + } +} + +/** The page item a list page's row names, or undefined for a page that lists none. */ +export function pageItemForListPath(pagePath: string, path: string): PageItemRef | undefined { + const clean = stripBase(pagePath) + if (clean === VARIABLES_PATH) return { kind: 'variable', path } + if (clean === RESOURCES_PATH) return { kind: 'resource', path } + if (clean === SCHEDULES_PATH) return { kind: 'schedule', path } + const trigger = Object.entries(TRIGGER_PAGES).find(([, p]) => p.path === clean) + return trigger ? { kind: 'trigger', triggerKind: trigger[0] as TriggerKind, path } : undefined +} + +const PAGE_ITEM_ROUTE = /^pageitem:(variable|resource|schedule|trigger\.([a-z]+))\/([^?#]+)$/ + +// A scheme rather than a path, like artifacts: the tab mounts the item's editor in process, +// so there is no page a frame could load. The path is encoded whole, so its slashes cannot +// be read as part of the scheme. +export function pageItemUrl(ref: PageItemRef): string { + const kind = ref.kind === 'trigger' ? `trigger.${ref.triggerKind}` : ref.kind + return `pageitem:${kind}/${encodeURIComponent(ref.path)}` +} + +export function parsePageItemRoute(url: string): PageItemRef | null { + const m = url.match(PAGE_ITEM_ROUTE) + if (!m) return null + let path: string + try { + path = decodeURIComponent(m[3]) + } catch { + return null + } + if (m[2] !== undefined) { + if (!(m[2] in TRIGGER_PAGES)) return null + return { kind: 'trigger', triggerKind: m[2] as TriggerKind, path } + } + return { kind: m[1] as 'variable' | 'resource' | 'schedule', path } +} + +/** Singular human name of a page item's kind, e.g. "Kafka trigger". */ +export function pageItemKindLabel(ref: PageItemRef): string { + switch (ref.kind) { + case 'variable': + return 'Variable' + case 'resource': + return 'Resource' + case 'schedule': + return 'Schedule' + case 'trigger': + return TRIGGER_PAGES[ref.triggerKind].label.replace(/s$/, '') + } +} + /** Label a trigger list page from its (base-stripped) pathname, or undefined. */ export function triggerLabelForPath(path: string): string | undefined { const clean = stripBase(path) diff --git a/frontend/src/lib/components/sessions/previewReload.test.ts b/frontend/src/lib/components/sessions/previewReload.test.ts index de11805b9a..ce9491d334 100644 --- a/frontend/src/lib/components/sessions/previewReload.test.ts +++ b/frontend/src/lib/components/sessions/previewReload.test.ts @@ -1,6 +1,7 @@ import { describe, it, expect } from 'vitest' import { toolReloadEffect, tabsToReload } from './previewReload' import type { SessionPreviewTab } from './sessionState.svelte' +import { pageItemUrl } from './previewPaths' describe('toolReloadEffect', () => { it('maps a non-item mutation to its own list page only', () => { @@ -52,6 +53,26 @@ describe('toolReloadEffect', () => { }) }) +describe('page item tabs', () => { + const tab = (url: string): SessionPreviewTab => ({ id: url, url, loc: url }) + const kafkaA = tab('pageitem:trigger.kafka/u%2Fme%2Fa') + const kafkaB = tab('pageitem:trigger.kafka/u%2Fme%2Fb') + const list = tab('/kafka_triggers') + + it('reloads only the trigger a write names, and its list page', () => { + const { pages, items } = toolReloadEffect('write_trigger', { + kind: 'kafka', + config: { path: 'u/me/a' } + }) + const named = new Set(items.map((i) => pageItemUrl(i))) + expect(tabsToReload([kafkaA, kafkaB, list], new Set(pages), named)).toEqual([kafkaA, list]) + }) + + it('reloads every tab of the kind when the tool names no item', () => { + expect(tabsToReload([kafkaA, kafkaB], new Set(['/kafka_triggers']))).toEqual([kafkaA, kafkaB]) + }) +}) + describe('tabsToReload', () => { const scheduleTab: SessionPreviewTab = { id: 's', url: '/schedules', loc: '/schedules' } const resourceTab: SessionPreviewTab = { id: 'r', url: '/resources', loc: '/resources' } diff --git a/frontend/src/lib/components/sessions/previewReload.ts b/frontend/src/lib/components/sessions/previewReload.ts index 6e9d750a1f..bdfbeaa3ff 100644 --- a/frontend/src/lib/components/sessions/previewReload.ts +++ b/frontend/src/lib/components/sessions/previewReload.ts @@ -1,6 +1,14 @@ import type { SessionPreviewTab } from './sessionState.svelte' import { whereIs } from './sessionPreviewTabs.svelte' -import { stripBase, TRIGGER_PAGES, type TriggerKind } from './previewPaths' +import { + pageItemListPath, + pageItemUrl, + parsePageItemRoute, + stripBase, + TRIGGER_PAGES, + type PageItemRef, + type TriggerKind +} from './previewPaths' // Which list pages a completed chat tool can change, as base-stripped paths // (e.g. `/schedules`). This allowlist is the single source of truth for "does @@ -13,23 +21,27 @@ import { stripBase, TRIGGER_PAGES, type TriggerKind } from './previewPaths' // deliberately absent: every editable item is a live in-process editor that // self-syncs from the store the chat mutates, so its tab needs no reload — and // no list page we preview lists open drafts. They fall through to NO_RELOAD. -// This "live editors self-sync, only list pages reload" invariant is the reason -// the callers below and in the sessions page reload nothing for item tabs. -export type ToolReloadEffect = { pages: string[] } -const NO_RELOAD: ToolReloadEffect = { pages: [] } +// +// Page items (variables, resources, schedules, triggers) are the exception among +// in-process tabs: their editors read a draft only when they open, so a write to +// one reloads its tab too. `items` names it when the tool's args do; without a +// path, every tab of that kind reloads. +export type ToolReloadEffect = { pages: string[]; items: PageItemRef[] } +const NO_RELOAD: ToolReloadEffect = { pages: [], items: [] } export function toolReloadEffect(name: string, args: any): ToolReloadEffect { switch (name) { case 'write_schedule': - return { pages: ['/schedules'] } + return withItem(['/schedules'], itemRef('schedule', args)) case 'write_trigger': - return { pages: triggerPages(args?.kind) } + // Its path sits in the trigger's own config, not beside `kind`. + return withItem(triggerPages(args?.kind), itemRef('trigger', args?.config, args?.kind)) case 'write_resource': - return { pages: ['/resources'] } + return withItem(['/resources'], itemRef('resource', args)) case 'write_variable': - return { pages: ['/variables'] } + return withItem(['/variables'], itemRef('variable', args)) case 'create_folder': - return { pages: ['/folders'] } + return { pages: ['/folders'], items: [] } // Generic item tools carry a workspace-item `type`; refresh its list page // when it lives on one (schedule/resource/variable/trigger). script/flow/app // have their own live editor tab and no previewed list page → nothing. @@ -37,12 +49,31 @@ export function toolReloadEffect(name: string, args: any): ToolReloadEffect { case 'discard_local_draft': case 'deploy_workspace_item': case 'rebase_draft': - return { pages: pagesForItemType(args?.type, args) } + return withItem( + pagesForItemType(args?.type, args), + itemRef(args?.type, args, args?.trigger_kind) + ) default: return NO_RELOAD } } +function withItem(pages: string[], item: PageItemRef | undefined): ToolReloadEffect { + return { pages, items: item && pages.length ? [item] : [] } +} + +function itemRef(type: unknown, args: any, triggerKind?: unknown): PageItemRef | undefined { + const path = args?.path + if (typeof path !== 'string' || !path) return undefined + if (type === 'variable' || type === 'resource' || type === 'schedule') { + return { kind: type, path } + } + if (type === 'trigger' && (triggerKind as string) in TRIGGER_PAGES) { + return { kind: 'trigger', triggerKind: triggerKind as TriggerKind, path } + } + return undefined +} + function pagesForItemType(type: unknown, args: any): string[] { switch (type) { case 'schedule': @@ -63,14 +94,23 @@ function triggerPages(kind: unknown): string[] { return page ? [page.path] : [] } -// The open tabs a page-reload should refresh: those whose observed page path is -// in `pages`. Item-editor and pipeline tab routes are never list pages, so they -// never match (see the self-sync invariant above). Pure over a tab snapshot so -// the sessions page can reload by id and this stays unit-testable. +// The open tabs a reload should refresh: list-page tabs whose observed page path is +// in `pages`, and page item tabs on those pages — only the named ones when a tool +// named its item. Item-editor and pipeline tab routes are never list pages, so they +// never match (see the self-sync invariant above). Pure over a tab snapshot so the +// sessions page can reload by id and this stays unit-testable. export function tabsToReload( tabs: SessionPreviewTab[], - pages: ReadonlySet + pages: ReadonlySet, + items: ReadonlySet = new Set() ): SessionPreviewTab[] { if (pages.size === 0) return [] - return tabs.filter((t) => pages.has(stripBase(whereIs(t)))) + return tabs.filter((t) => { + const pageItem = parsePageItemRoute(t.url) + if (!pageItem) return pages.has(stripBase(whereIs(t))) + const listPath = pageItemListPath(pageItem) + if (!pages.has(listPath)) return false + const named = [...items].some((u) => pageItemListPath(parsePageItemRoute(u)!) === listPath) + return !named || items.has(pageItemUrl(pageItem)) + }) } diff --git a/frontend/src/lib/components/sessions/previewRouter.test.ts b/frontend/src/lib/components/sessions/previewRouter.test.ts index 7d56f9e717..3bc76882e5 100644 --- a/frontend/src/lib/components/sessions/previewRouter.test.ts +++ b/frontend/src/lib/components/sessions/previewRouter.test.ts @@ -14,8 +14,27 @@ import { previewLocationContext, previewLocationLabel, resolvePreviewTab, - runFormUrl + runFormUrl, + workspacePageHref } from './previewRouter' +import { pageItemUrl } from './previewPaths' + +describe('workspacePageHref', () => { + it('sends a page item tab to its list page with the row open, never to its scheme', () => { + expect(workspacePageHref(pageItemUrl({ kind: 'resource', path: 'u/me/db' }))).toBe( + '/resources#/resource/u/me/db' + ) + expect( + workspacePageHref(pageItemUrl({ kind: 'trigger', triggerKind: 'kafka', path: 'f/a/b' })) + ).toBe('/kafka_triggers#f/a/b') + }) + + it('has no page for a tab that belongs to the chat', () => { + expect(workspacePageHref(artifactUrl('a1', 'Plan'))).toBeUndefined() + expect(workspacePageHref(runFormUrl('call_1', 'Run'))).toBeUndefined() + expect(workspacePageHref('/runs?path=u/me/x')).toBe('/runs?path=u/me/x') + }) +}) describe('drawerAnchorFor', () => { it('reads the anchored row on the pages that deep-link one', () => { diff --git a/frontend/src/lib/components/sessions/previewRouter.ts b/frontend/src/lib/components/sessions/previewRouter.ts index 9f312ea389..13e7626e70 100644 --- a/frontend/src/lib/components/sessions/previewRouter.ts +++ b/frontend/src/lib/components/sessions/previewRouter.ts @@ -3,8 +3,13 @@ import { AUDIT_LOGS_PATH, FOLDERS_PATH, GROUPS_PATH, + pageItemForListPath, + pageItemListPath, + pageItemPageHref, + pageItemUrl, pageKey, pageHref, + parsePageItemRoute, parsePreviewItemRoute, RESOURCES_PATH, RUNS_PATH, @@ -14,17 +19,22 @@ import { WORKSPACE_SETTINGS_PATH, triggerLabelForPath, TRIGGER_PAGES, + type PageItemRef, type PreviewItemRoute, type TriggerKind } from './previewPaths' // Re-exported so the preview code that already reads locations through this module keeps // one import, while a caller needing only a path can reach for the leaf instead. export { + pageItemListPath, + pageItemUrl, pageKey, pageHref, + parsePageItemRoute, parsePreviewItemRoute, stripBase, TRIGGER_PAGES, + type PageItemRef, type PreviewItemRoute, type TriggerKind } @@ -68,6 +78,7 @@ export type PreviewTarget = | { type: 'item'; item: WorkspaceItem } | { type: 'artifact'; id: string; name: string; version?: ArtifactVersionTarget } | { type: 'runform'; toolCallId: string; label: string } + | { type: 'pageitem'; ref: PageItemRef } export type PreviewPage = { label: string; path: string; icon: DrillIcon } @@ -117,6 +128,26 @@ export function drawerAnchorFor(location: string): string | undefined { return location.slice(hashAt + 1).replace(/^\/resource\//, '') || undefined } +/** The item a list-page location deep-links, as a tab of its own: a session edits these + * in process, so the list page's drawer is never where one belongs. */ +export function pageItemForLocation(location: string): PageItemRef | undefined { + const anchor = drawerAnchorFor(location) + if (!anchor) return undefined + let path: string + try { + path = decodeURIComponent(anchor) + } catch { + return undefined + } + return pageItemForListPath(location, path) +} + +/** A location with a deep-linked row replaced by that row's own tab; any other unchanged. */ +export function pageItemLocation(location: string): string { + const ref = pageItemForLocation(location) + return ref ? pageItemUrl(ref) : location +} + // Query params the preview host injects into an iframe URL (`nomenubar` hides the nav, // `workspace` scopes the page). Never part of what a location means. const INJECTED_PARAMS = ['nomenubar', 'workspace'] as const @@ -126,7 +157,7 @@ const INJECTED_PARAMS = ['nomenubar', 'workspace'] as const export function canonicalizeObservedLoc(loc: string): string { // An artifact or a run form is a scheme, not a path — `new URL` would happily parse it // and hand back a pathname with the scheme gone. - if (parseArtifactRoute(loc) || parseRunFormRoute(loc)) return loc + if (parseArtifactRoute(loc) || parseRunFormRoute(loc) || parsePageItemRoute(loc)) return loc try { const u = new URL(loc, 'http://_') for (const p of INJECTED_PARAMS) u.searchParams.delete(p) @@ -201,6 +232,8 @@ export function describeLocation(loc: string): PreviewLocation { // Identity is the call, never the label: that carries the script's summary, so folding it // in would open a second tab for the same form whenever the summary differed. if (runForm) return { identity: `runform:${runForm.toolCallId}`, view: '', anchor: '' } + const pageItem = parsePageItemRoute(loc) + if (pageItem) return { identity: pageItemUrl(pageItem), view: '', anchor: '' } const canonical = canonicalizeObservedLoc(loc) const path = stripBase(canonical) const bare = canonical.split('#')[0] @@ -322,6 +355,15 @@ export function previewLocationContext(loc: string): { location: string open?: string } { + // Told as its list page with the item open, the shape the model already reads for a row + // whose drawer is open — which is all a page item tab is to it. + const pageItem = parsePageItemRoute(loc) + if (pageItem) { + return { + ...previewLocationContext(pageItemListPath(pageItem)), + open: promptSafe(pageItem.path) + } + } const { identity, anchor } = describeLocation(loc) const bare = canonicalizeObservedLoc(loc).split('#')[0] const query = bare.includes('?') ? bare.slice(bare.indexOf('?') + 1) : '' @@ -370,6 +412,8 @@ export function previewLocationLabel(url: string): string { if (artifact) return artifact.name || 'Artifact' const runForm = parseRunFormRoute(url) if (runForm) return runForm.label || 'Run form' + const pageItem = parsePageItemRoute(url) + if (pageItem) return pageItem.path.split('/').pop() || pageItem.path const page = matchReusablePage(url) if (page) return page.label const trigger = triggerLabelForPath(url) @@ -492,6 +536,7 @@ export type PreviewSlot = | { kind: 'editor'; editorKind: SessionTargetKind | 'pipeline'; path: string } | { kind: 'artifact'; id: string; version?: number } | { kind: 'runform'; toolCallId: string } + | { kind: 'pageitem'; ref: PageItemRef } | { kind: 'iframe' } export function resolvePreviewTab(url: string): PreviewSlot { @@ -499,6 +544,8 @@ export function resolvePreviewTab(url: string): PreviewSlot { if (artifact) return { kind: 'artifact', id: artifact.id, version: artifact.version } const runForm = parseRunFormRoute(url) if (runForm) return { kind: 'runform', toolCallId: runForm.toolCallId } + const pageItem = parsePageItemRoute(url) + if (pageItem) return { kind: 'pageitem', ref: pageItem } const pipelineFolder = parsePipelineRoute(url) if (pipelineFolder) { return { kind: 'editor', editorKind: 'pipeline', path: pipelineFolder } @@ -516,3 +563,24 @@ export function resolvePreviewTab(url: string): PreviewSlot { if (!editorKind) return { kind: 'iframe' } return { kind: 'editor', editorKind, path: route.itemPath } } + +/** The full workspace page showing what a tab shows ("Open in workspace"), or undefined when + * there is none. Every tab kind answers here, so a new one cannot fall through to its url being + * navigated as a path — an artifact or page item url is a scheme, not a route. */ +export function workspacePageHref(location: string): string | undefined { + const slot = resolvePreviewTab(location) + switch (slot.kind) { + case 'artifact': + case 'runform': + return undefined + case 'pageitem': + return pageItemPageHref(slot.ref) + case 'editor': + case 'iframe': + return location + default: { + const unhandled: never = slot + return unhandled + } + } +} diff --git a/frontend/src/lib/components/sessions/sessionMirror.svelte.ts b/frontend/src/lib/components/sessions/sessionMirror.svelte.ts index bfa1621822..f30d66ca45 100644 --- a/frontend/src/lib/components/sessions/sessionMirror.svelte.ts +++ b/frontend/src/lib/components/sessions/sessionMirror.svelte.ts @@ -1169,7 +1169,6 @@ function unpackBackup( const active = chats.find((c) => c.id === head.chatId) const session: Session = { ...(head as unknown as Session), - name: '', // Everything in the backup has been read here: no unread badge, and the last // activity is the backup's own time. lastSeenCount: active?.displayMessages.length ?? 0, diff --git a/frontend/src/lib/components/sessions/sessionMirror.test.ts b/frontend/src/lib/components/sessions/sessionMirror.test.ts index 7676662d31..55a1acc138 100644 --- a/frontend/src/lib/components/sessions/sessionMirror.test.ts +++ b/frontend/src/lib/components/sessions/sessionMirror.test.ts @@ -1077,7 +1077,7 @@ describe('sessionMirror restore', () => { expect(pullMock).toHaveBeenCalledTimes(2) await vi.waitFor(() => expect(sessionState.sessions.map((s) => s.id)).toEqual(['s9'])) const restored = sessionState.sessions[0] - expect(restored.name).toBe('session-1') + expect(restored.name).toBeUndefined() expect(restored.summary).toBe('remote') expect(restored.lastSeenCount).toBe(1) expect((await readStoredChat('c9', EMAIL))?.displayMessages).toHaveLength(1) diff --git a/frontend/src/lib/components/sessions/sessionMirrorPlan.ts b/frontend/src/lib/components/sessions/sessionMirrorPlan.ts index caea48c792..5d40c35e3b 100644 --- a/frontend/src/lib/components/sessions/sessionMirrorPlan.ts +++ b/frontend/src/lib/components/sessions/sessionMirrorPlan.ts @@ -78,7 +78,7 @@ export function isFallbackStorage(name: string): boolean { /** * The part of a session record the backup keeps. Left out on purpose: `name` (a - * per-browser counter), the unsent-draft fields (`pending_*`, `draftPrompt`, + * per-browser legacy label), the unsent-draft fields (`pending_*`, `draftPrompt`, * `autoSendDraftAt`), `workspace_root_id` (derived on import), `transient`, `restoredAt` * (this browser's clock), and the two fields reading a session bumps (`lastSeenCount`, * `lastActivityAt`) — so opening a session and reading its new messages never costs a push. diff --git a/frontend/src/lib/components/sessions/sessionMode.svelte.ts b/frontend/src/lib/components/sessions/sessionMode.svelte.ts index a4972a9afb..a2c20e08a8 100644 --- a/frontend/src/lib/components/sessions/sessionMode.svelte.ts +++ b/frontend/src/lib/components/sessions/sessionMode.svelte.ts @@ -47,6 +47,17 @@ export function withMenuHidden(url: string, workspaceId?: string): string { } } +// True when this window is a sessions-preview iframe: embedded, with the `nomenubar` flag +// the preview always sets and the logged layout stickies into sessionStorage. +export function isSessionPreviewFrame(): boolean { + if (typeof window === 'undefined' || window.self === window.top) return false + try { + return sessionStorage.getItem('nomenubar_embedded') === 'true' + } catch { + return false + } +} + // Append `?workspace=` to a canonical route so a full-page navigation (e.g. // "Open in workspace") lands on the session's effective workspace instead of // the navigation workspace. Unlike withMenuHidden, the menu is kept visible — diff --git a/frontend/src/lib/components/sessions/sessionPreviewTabs.svelte.ts b/frontend/src/lib/components/sessions/sessionPreviewTabs.svelte.ts index a9db01e0fa..a90eb83cc4 100644 --- a/frontend/src/lib/components/sessions/sessionPreviewTabs.svelte.ts +++ b/frontend/src/lib/components/sessions/sessionPreviewTabs.svelte.ts @@ -8,7 +8,10 @@ import { describeLocation, matchPreviewPage, showsView, + pageItemLocation, + pageItemUrl, parseArtifactRoute, + parsePageItemRoute, parsePipelineRoute, previewLocationContext, promptSafe, @@ -24,6 +27,12 @@ import { import type { SessionPreviewTab, SessionTarget } from './sessionState.svelte' import type { Kind } from '$lib/utils_deployable' import { pipelineFolderFromBundlePath } from '$lib/pipelinePaths' +import { + pageItemKindLabel, + TRIGGER_PAGES, + type PageItemRef, + type TriggerKind +} from './previewPaths' // The single live owner of a session's preview tabs. Runs behind a small // interface both the sessions page (renderer) and the `open_preview` tool cross, @@ -78,7 +87,10 @@ function keptVersion( // scheme. `onto` is the tab about to be written, passed wherever one is being re-pointed so // that every such path keeps its pin. function targetUrl(target: PreviewTarget, onto?: SessionPreviewTab): string { - if (target.type === 'page') return target.href + // A list page asked for with a row anchored is that row's own tab: its drawer would only + // open the editor a page item tab already hosts, inside a frame of its own. + if (target.type === 'page') return pageItemLocation(target.href) + if (target.type === 'pageitem') return pageItemUrl(target.ref) if (target.type === 'artifact') { return artifactUrl(target.id, target.name, keptVersion(target, onto)) } @@ -152,10 +164,9 @@ export function previewTargetForSessionTarget( // Adapt a deployable item's layout kind (the session review dock speaks `Kind`, // not SessionTarget) to a preview destination: the three live editors, data -// pipelines, plus legacy drag-and-drop apps, which the panel hosts as an iframe -// over their edit route. Every other kind maps to undefined — not for lack of any -// route (a variable or trigger has a list page the panel can host) but because -// there is no item editor to preview, so their row falls back to the diff. The +// pipelines, page items (variables, resources, schedules, triggers), plus legacy +// drag-and-drop apps, which the panel hosts as an iframe over their edit route. +// Every other kind maps to undefined, and its row falls back to the diff. The // undefined is also the caller's test for "can this row be previewed?". export function previewTargetForDeployKind(kind: Kind, path: string): PreviewTarget | undefined { if (kind === 'app') { @@ -164,6 +175,16 @@ export function previewTargetForDeployKind(kind: Kind, path: string): PreviewTar if (kind === 'script' || kind === 'flow' || kind === 'raw_app') { return previewTargetForSessionTarget(kind, path) } + if (kind === 'variable' || kind === 'resource' || kind === 'schedule') { + return { type: 'pageitem', ref: { kind, path } } + } + const triggerKind = kind.endsWith('_trigger') ? kind.slice(0, -'_trigger'.length) : undefined + if (triggerKind && triggerKind in TRIGGER_PAGES) { + return { + type: 'pageitem', + ref: { kind: 'trigger', triggerKind: triggerKind as TriggerKind, path } + } + } // A pipeline's editor is its folder's graph view, not its bundle path. if (kind === 'data_pipeline') { const folder = pipelineFolderFromBundlePath(path) @@ -189,7 +210,14 @@ export function hydratePreviewTabs(session: { seen.add(t.id) // Rebuilt field-by-field so stray properties on old saved records (e.g. the // retired `pinned` flag) don't survive hydration and get persisted back. - tabs.push({ id: t.id, url: t.url, loc: t.loc || t.url }) + // A list page saved with a row's drawer open comes back as that row's own tab. Read from + // where the frame was, not from what was last commanded: the user may have moved to + // another row since, or closed the drawer, and the observation is what they saw. + const loc = t.loc || t.url + const item = pageItemLocation(t.loc ? loc : t.url) + tabs.push( + parsePageItemRoute(item) ? { id: t.id, url: item, loc: item } : { id: t.id, url: t.url, loc } + ) } if (tabs.length > 0) { const wantActive = session.activePreviewTabId @@ -290,22 +318,14 @@ export class SessionPreviewTabs { // Drift is a change of what the frame *shows*, not of its URL string: a page // writing its own filter defaults back is not the user navigating away. const drifted = !showsView(tab.loc, url) - // Both cases the browser will not act on, decided here because this is where the - // old and new commands are both in hand: re-commanding the URL a drifted frame - // already carries moves nothing, and moving to another fragment resolves within the - // same document — so a list page never re-runs the `#` read that opens a row. - // Dropping the fragment is not one of them: the same-document path applies only to a - // target that has one, so the browser loads the page — closing the drawer by itself — - // and forcing a second load races that one back onto the row. - const fragmentOnly = - !commandUnchanged && url.includes('#') && tab.url.split('#')[0] === url.split('#')[0] + // Decided here because this is where the old and new commands are both in hand: + // re-commanding the URL a drifted frame already carries moves nothing. retargetTab(tab, url) - if ((commandUnchanged && drifted) || fragmentOnly) this.pulseReload(tab.id) + if (commandUnchanged && drifted) this.pulseReload(tab.id) } - // Force the host to reload the iframe. A navigation onto the tab's exact current URL - // changes nothing, so URL-driven behavior — a `#` opening a drawer the user has - // since closed — would never re-fire. + // Force the host to reload the tab. A navigation onto the tab's exact current URL + // changes nothing, so URL-driven behavior would never re-fire. pulseReload(id: string): void { this.#reloadPulse = { id, nonce: this.#reloadPulse.nonce + 1 } } @@ -435,21 +455,21 @@ export class SessionPreviewTabs { // shows this. The tab on this exact view wins over any other on the page — // `new_tab` puts two views side by side, and retargeting whichever sits first // would overwrite the other and leave both on the same row. - const shown = opts?.forceNewTab - ? undefined - : (this.#tabs.find((t) => showsView(t.loc, url)) ?? - this.#tabs.find((t) => describeLocation(t.loc).identity === describeLocation(url).identity)) + // A page item stays one tab whatever the opener asks: two would hold two drafts of it. + const shown = + opts?.forceNewTab && !parsePageItemRoute(url) + ? undefined + : (this.#tabs.find((t) => showsView(t.loc, url)) ?? + this.#tabs.find( + (t) => describeLocation(t.loc).identity === describeLocation(url).identity + )) if (shown) { const same = showsView(shown.loc, url) if (same) { // The frame is already here, but record what was asked for: `url` is what the // tab persists and remounts from, so leaving it on where the frame started - // sends a refresh back to the row the user has since moved off. + // sends a refresh back to the view the user has since moved off. recordCommand(shown, url) - // Nothing to navigate to, so nothing would re-run: the list pages read their - // `#` once per document, and the drawer it opens may since have been - // closed. Only a forced load can bring it back. - if (describeLocation(url).anchor) this.pulseReload(shown.id) } else { this.#retarget(shown, url) } @@ -512,7 +532,17 @@ export class SessionPreviewTabs { return } } - this.#retarget(t, targetUrl(target, t)) + // One tab per page item, as for editors: two would hold two drafts of one item. + const url = targetUrl(target, t) + if (parsePageItemRoute(url)) { + const existing = this.#tabs.find((x) => x.url === url) + if (existing && existing.id !== t.id) { + this.#activeId = existing.id + this.#flush() + return + } + } + this.#retarget(t, url) this.#flush() } @@ -574,6 +604,17 @@ export class SessionPreviewTabs { this.#flush() } + /** Follow a page item its editor saved under a new path, in place. */ + retargetPageItem(from: PageItemRef, to: PageItemRef): void { + const fromUrl = pageItemUrl(from) + const toUrl = pageItemUrl(to) + if (fromUrl === toUrl) return + const tab = this.#tabs.find((t) => t.url === fromUrl) + if (!tab) return + retargetTab(tab, toUrl) + this.#flush() + } + closeArtifact(artifactId: string): void { const tab = this.#tabs.find((t) => parseArtifactRoute(t.url)?.id === artifactId) if (tab) this.close(tab.id) @@ -606,23 +647,14 @@ export class SessionPreviewTabs { } // Feed back the location an iframe reported on load (only the page can read - // contentWindow.location). Updates the observed `loc`; `url` follows only when a - // drawer closed (below), and the host navigates on a command it isn't already at, - // so that write does not move the frame. + // contentWindow.location). Updates the observed `loc` only: the host navigates on a + // command it isn't already at, and an in-frame move is the user browsing. observeLocation(id: string, loc: string): void { const t = this.#tabs.find((x) => x.id === id) if (!t) return const canonical = canonicalizeObservedLoc(loc) if (t.loc === canonical) return t.loc = canonical - // Closing a drawer drops the row from the frame's URL. The command has to follow, or - // the tab reopens it on the next mount — the iframe loads `url`, not `loc`. Only the - // anchor: any other in-frame move is the user browsing, which must not re-command. - const commanded = describeLocation(t.url) - const observed = describeLocation(canonical) - if (commanded.anchor && !observed.anchor && commanded.identity === observed.identity) { - t.url = t.url.split('#')[0] - } this.#flush() } @@ -730,6 +762,7 @@ export function describePreview( const lines = tabs.map((t) => { const where = whereIs(t) const artifact = parseArtifactRoute(where) + const pageItem = parsePageItemRoute(where) const page = matchPreviewPage(where) const pipelineFolder = parsePipelineRoute(where) const route = parsePreviewItemRoute(where) @@ -737,16 +770,19 @@ export function describePreview( ? // A pinned tab is not showing what the assistant last wrote, and nothing else in this // summary would tell it so. `artifact "${artifact.name || 'Artifact'}"${artifact.version ? ` (pinned to v${artifact.version})` : ''}` - : page - ? `page "${page.label}"${previewLocationDetail(where)}` - : pipelineFolder - ? `pipeline "${pipelineFolder}"` - : route - ? `${route.raw_app ? 'raw_app' : route.kind} "${route.itemPath}"` - : // Trigger list pages land here (they're outside PREVIEW_PAGES), and - // their `#` is the trigger the drawer has open. - `${stripBase(where)}${previewLocationDetail(where)}` - const live = resolvePreviewTab(t.url).kind === 'editor' ? ', live editor' : '' + : pageItem + ? `${pageItemKindLabel(pageItem).toLowerCase()} "${pageItem.path}"` + : page + ? `page "${page.label}"${previewLocationDetail(where)}` + : pipelineFolder + ? `pipeline "${pipelineFolder}"` + : route + ? `${route.raw_app ? 'raw_app' : route.kind} "${route.itemPath}"` + : // Trigger list pages land here (they're outside PREVIEW_PAGES), and + // their `#` is the trigger the drawer has open. + `${stripBase(where)}${previewLocationDetail(where)}` + const slotKind = resolvePreviewTab(t.url).kind + const live = slotKind === 'editor' || slotKind === 'pageitem' ? ', live editor' : '' const active = t.id === activeId ? ', active' : '' // One list entry per tab: an artifact's name, a pipeline folder and an item path // all arrive decoded from a URL, so any of them could otherwise write a line here. diff --git a/frontend/src/lib/components/sessions/sessionPreviewTabs.test.ts b/frontend/src/lib/components/sessions/sessionPreviewTabs.test.ts index b71de62c23..8b1ba27ea0 100644 --- a/frontend/src/lib/components/sessions/sessionPreviewTabs.test.ts +++ b/frontend/src/lib/components/sessions/sessionPreviewTabs.test.ts @@ -177,10 +177,88 @@ describe('previewTargetForDeployKind', () => { pipelineTarget ) }) + it('routes variables, resources, schedules and triggers to their own tab', () => { + expect(previewTargetForDeployKind('schedule', 'u/me/s')).toEqual({ + type: 'pageitem', + ref: { kind: 'schedule', path: 'u/me/s' } + }) + expect(previewTargetForDeployKind('http_trigger', 'u/me/t')).toEqual({ + type: 'pageitem', + ref: { kind: 'trigger', triggerKind: 'http', path: 'u/me/t' } + }) + expect(previewTargetForDeployKind('variable', 'u/me/v')).toEqual({ + type: 'pageitem', + ref: { kind: 'variable', path: 'u/me/v' } + }) + }) + it('has no destination for kinds the preview panel cannot host', () => { - expect(previewTargetForDeployKind('schedule', 'u/me/s')).toBeUndefined() - expect(previewTargetForDeployKind('http_trigger', 'u/me/t')).toBeUndefined() - expect(previewTargetForDeployKind('variable', 'u/me/v')).toBeUndefined() + expect(previewTargetForDeployKind('folder', 'f/x')).toBeUndefined() + expect(previewTargetForDeployKind('resource_type', 'x')).toBeUndefined() + }) +}) + +describe('page item tabs', () => { + const variable: PreviewTarget = { + type: 'pageitem', + ref: { kind: 'variable', path: 'u/me/token' } + } + + it('opens a list page anchored at a row as that row’s own tab, beside the list', () => { + const o = owner() + o.open({ type: 'page', href: '/routes', label: 'HTTP routes' }) + o.open({ type: 'page', href: '/routes#u/me/a', label: 'HTTP routes' }) + expect(o.tabs.map((t) => t.url)).toEqual(['/routes', 'pageitem:trigger.http/u%2Fme%2Fa']) + + // Resources address their row through an extra segment. + o.open({ type: 'page', href: '/resources?owner=u#/resource/u/me/db', label: 'Resources' }) + expect(o.tabs.at(-1)!.url).toBe('pageitem:resource/u%2Fme%2Fdb') + }) + + it('keeps one tab per item, whatever the opener asks', () => { + const o = owner() + o.open(variable) + o.open({ type: 'page', href: '/runs', label: 'Runs' }) + expect(o.open({ type: 'page', href: '/variables#u/me/token', label: 'V' }).status).toBe( + 'focused' + ) + expect(o.open(variable, { forceNewTab: true }).status).toBe('focused') + o.navigate(variable) + expect(o.tabs).toHaveLength(2) + expect(o.activeId).toBe(o.tabs[0].id) + }) + + it('follows an item saved under a new path in place', () => { + const o = owner() + o.open(variable) + const id = o.tabs[0].id + o.retargetPageItem( + { kind: 'variable', path: 'u/me/token' }, + { kind: 'variable', path: 'f/x/token' } + ) + expect(o.tabs).toEqual([ + { id, url: 'pageitem:variable/f%2Fx%2Ftoken', loc: 'pageitem:variable/f%2Fx%2Ftoken' } + ]) + }) + + it('restores a tab saved on a row’s drawer as the row it was last on', () => { + const snap = hydratePreviewTabs({ + previewTabs: [ + { id: 'a', url: '/schedules#u/me/daily', loc: '/schedules?path=u#u/me/daily' }, + // A drawer opened inside the frame: the command is still the bare list. + { id: 'b', url: '/variables', loc: '/variables#u/me/token' }, + // The user moved on to another row inside the frame… + { id: 'c', url: '/variables#u/me/token', loc: '/variables#u/me/other' }, + // …or closed the drawer, leaving no row to restore. + { id: 'd', url: '/variables#u/me/token', loc: '/variables' } + ] + }) + expect(snap.tabs).toEqual([ + { id: 'a', url: 'pageitem:schedule/u%2Fme%2Fdaily', loc: 'pageitem:schedule/u%2Fme%2Fdaily' }, + { id: 'b', url: 'pageitem:variable/u%2Fme%2Ftoken', loc: 'pageitem:variable/u%2Fme%2Ftoken' }, + { id: 'c', url: 'pageitem:variable/u%2Fme%2Fother', loc: 'pageitem:variable/u%2Fme%2Fother' }, + { id: 'd', url: '/variables#u/me/token', loc: '/variables' } + ]) }) }) @@ -232,93 +310,34 @@ describe('SessionPreviewTabs.open', () => { expect(o.activeId).toBe(firstId) }) - // A trigger list page is not a `matchReusablePage`, so the runtime's - // navigate-in-place path doesn't cover it: re-pointing the tab has to happen - // here or the panel keeps showing the previously opened row. - it('re-points a page tab whose hash target changed instead of only focusing it', () => { - const o = owner() - const routes = (href: string) => ({ type: 'page' as const, href, label: 'HTTP routes' }) - o.open(routes('/routes#u/me/a')) - const firstId = o.activeId - - // 'retargeted', not 'opened': the tab count is unchanged, and the caller - // reports that to the model. - const res = o.open(routes('/routes#u/me/b')) - expect(res.status).toBe('retargeted') - expect(o.tabs).toHaveLength(1) - expect(o.activeId).toBe(firstId) - expect(o.tabs[0].url).toBe('/routes#u/me/b') - - // Back to the bare list: still the same tab, no longer anchored at a row. - expect(o.open(routes('/routes')).status).toBe('retargeted') - expect(o.tabs).toHaveLength(1) - expect(o.tabs[0].url).toBe('/routes') - - // ...and asking for the view it already shows is a plain focus. - expect(o.open(routes('/routes')).status).toBe('focused') - }) - // The list pages rewrite their own filter defaults into the URL after mount, // and `loc` follows that rewrite. Matching on anything but the path made a tab // stop recognizing itself, so every later open spawned a duplicate. it('still recognizes a tab after the page rewrote its own filter params', () => { const o = owner() const routes = (href: string) => ({ type: 'page' as const, href, label: 'HTTP routes' }) - o.open(routes('/routes#u/me/a')) - const id = o.tabs[0].id - o.observeLocation(id, '/routes?filter_path_of=trigger#u/me/a') + o.open(routes('/routes')) + o.observeLocation(o.tabs[0].id, '/routes?filter_path_of=trigger') - const res = o.open(routes('/routes#u/me/b')) - expect(res.status).toBe('retargeted') + expect(o.open(routes('/routes')).status).toBe('focused') expect(o.tabs).toHaveLength(1) - expect(o.tabs[0].url).toBe('/routes#u/me/b') }) // `new_tab` deliberately keeps two views of one page side by side. Reopening one of // them must focus the tab already showing it, not retarget whichever tab happens to - // sit first in the strip — that would overwrite the other view and leave two tabs - // on the same row. + // sit first in the strip — that would overwrite the other view. it('focuses the tab already showing the exact location before retargeting by path', () => { const o = owner() - const routes = (href: string) => ({ type: 'page' as const, href, label: 'HTTP routes' }) - o.open(routes('/routes#u/me/a')) + const runs = (href: string) => ({ type: 'page' as const, href, label: 'Runs' }) + o.open(runs('/runs?path=u/me/a')) const first = o.tabs[0].id - o.open(routes('/routes#u/me/b'), { forceNewTab: true }) + o.open(runs('/runs?path=u/me/b'), { forceNewTab: true }) const second = o.tabs[1].id - expect(o.open(routes('/routes#u/me/b')).status).toBe('focused') + expect(o.open(runs('/runs?path=u/me/b')).status).toBe('focused') expect(o.activeId).toBe(second) expect(o.tabs).toHaveLength(2) - expect(o.tabs.find((t) => t.id === first)?.url).toBe('/routes#u/me/a') - }) - - // The list pages read their `#` once per document, so a drawer the user closed - // inside the frame only comes back on a forced load — and re-commanding the location - // the tab already shows produces no navigation the host could act on. - it('forces a load when the requested row is the one the tab already shows', () => { - const o = owner() - const routes = (href: string) => ({ type: 'page' as const, href, label: 'HTTP routes' }) - o.open(routes('/routes#u/me/a')) - const id = o.tabs[0].id - o.observeLocation(id, '/routes?filter_path_of=trigger#u/me/a') - const before = o.reloadPulse.nonce - - expect(o.open(routes('/routes#u/me/a')).status).toBe('focused') - expect(o.reloadPulse).toEqual({ id, nonce: before + 1 }) - }) - - // Dropping the fragment is a load in itself, so the forced one lands on top of a - // navigation still in flight — and reloads the row the command asked to leave. - it('does not force a load when the requested location drops the row', () => { - const o = owner() - const routes = (href: string) => ({ type: 'page' as const, href, label: 'HTTP routes' }) - o.open(routes('/routes#u/me/a')) - const id = o.tabs[0].id - const before = o.reloadPulse.nonce - - o.navigate(routes('/routes')) - expect(o.tabs.find((t) => t.id === id)?.url).toBe('/routes') - expect(o.reloadPulse.nonce).toBe(before) + expect(o.tabs.find((t) => t.id === first)?.url).toBe('/runs?path=u/me/a') }) // Runs restores the user's "hide schedules" preference into the URL whenever a load @@ -365,29 +384,11 @@ describe('SessionPreviewTabs.open', () => { expect(o.tabs[0].url).toBe('/apps/edit/u/me/dash') }) - // Re-commanding the URL a tab is already pointed at changes nothing the host can - // see, so the frame would stay wherever the user navigated it inside the page. - it('forces a reload when the request matches the command but the frame drifted', () => { - const o = owner() - const routes = (href: string) => ({ type: 'page' as const, href, label: 'HTTP routes' }) - o.open(routes('/routes#u/me/a')) - const id = o.tabs[0].id - // The user clicked another trigger inside the iframe. - o.observeLocation(id, '/routes#u/me/b') - const before = o.reloadPulse.nonce - - const res = o.open(routes('/routes#u/me/a')) - expect(res.status).toBe('retargeted') - expect(o.tabs).toHaveLength(1) - expect(o.tabs[0].loc).toBe('/routes#u/me/a') - expect(o.reloadPulse.nonce).toBe(before + 1) - }) - it('forceNewTab opts a page out of the location dedupe', () => { const o = owner() const routes = (href: string) => ({ type: 'page' as const, href, label: 'HTTP routes' }) - o.open(routes('/routes#u/me/a')) - const res = o.open(routes('/routes#u/me/b'), { forceNewTab: true }) + o.open(routes('/routes')) + const res = o.open(routes('/routes'), { forceNewTab: true }) expect(res.status).toBe('opened') expect(o.tabs).toHaveLength(2) }) @@ -512,59 +513,6 @@ describe('SessionPreviewTabs.open', () => { }) }) -describe('SessionPreviewTabs.open — commanded url', () => { - it('records the requested row even when the frame is already showing it', () => { - const o = owner() - o.open({ type: 'page', href: '/routes#u/me/a', label: 'R' }) - // The user moves to another row inside the frame. - o.observeLocation(o.tabs[0].id, '/routes#u/me/b') - o.open({ type: 'page', href: '/routes#u/me/b', label: 'R' }) - // `url` is what a refresh and a remount reload from, so it has to follow. - expect(o.tabs[0].url).toBe('/routes#u/me/b') - expect(o.tabs).toHaveLength(1) - }) -}) - -describe('SessionPreviewTabs.observeLocation', () => { - it('drops the row from the command when the frame closes its drawer', () => { - const o = owner() - o.open({ type: 'page', href: '/routes#u/me/a', label: 'R' }) - // The page clears its own hash when the drawer closes. - o.observeLocation(o.tabs[0].id, '/routes?filter_path_of=trigger') - // The iframe mounts from `url`, so a remount would otherwise reopen the drawer. - expect(o.tabs[0].url).toBe('/routes') - }) - - it('leaves the command alone when the user just browses inside the frame', () => { - const o = owner() - o.open({ type: 'page', href: '/routes#u/me/a', label: 'R' }) - o.observeLocation(o.tabs[0].id, '/routes#u/me/b') - expect(o.tabs[0].url).toBe('/routes#u/me/a') - }) -}) - -describe('SessionPreviewTabs.open — forced loads', () => { - it('pulses when only the fragment changes, since the browser would not load', () => { - const o = owner() - o.open({ type: 'page', href: '/routes#u/me/a', label: 'R' }) - const before = o.reloadPulse.nonce - o.open({ type: 'page', href: '/routes#u/me/b', label: 'R' }) - // Same document: the browser resolves the new fragment without a load, so the - // list page never re-runs the `#` read that opens the row. - expect(o.reloadPulse.nonce).toBeGreaterThan(before) - expect(o.tabs).toHaveLength(1) - }) - - it('does not pulse when the document itself changes', () => { - const o = owner() - o.open({ type: 'page', href: '/routes#u/me/a', label: 'R' }) - const before = o.reloadPulse.nonce - o.open({ type: 'page', href: '/schedules#u/me/a', label: 'S' }) - // Different page: src changes, the browser loads it, nothing to force. - expect(o.reloadPulse.nonce).toBe(before) - }) -}) - describe('SessionPreviewTabs.navigate', () => { it('retargets the active tab to an editor item', () => { const o = owner() diff --git a/frontend/src/lib/components/sessions/sessionRecoveryNotice.svelte.ts b/frontend/src/lib/components/sessions/sessionRecoveryNotice.svelte.ts index 635bd68601..443f972334 100644 --- a/frontend/src/lib/components/sessions/sessionRecoveryNotice.svelte.ts +++ b/frontend/src/lib/components/sessions/sessionRecoveryNotice.svelte.ts @@ -1,6 +1,6 @@ import { SvelteSet } from 'svelte/reactivity' -// Session ids opened as a stand-in for a `session_name` this browser doesn't +// Session ids opened as a stand-in for a `session` URL this browser doesn't // hold. Kept in memory rather than on the Session record: persisted, the notice // would replay on every reload of a session the user has since made their own. const recovered = new SvelteSet() diff --git a/frontend/src/lib/components/sessions/sessionRuntime.svelte.ts b/frontend/src/lib/components/sessions/sessionRuntime.svelte.ts index cca8c7cdfb..fbb84e1431 100644 --- a/frontend/src/lib/components/sessions/sessionRuntime.svelte.ts +++ b/frontend/src/lib/components/sessions/sessionRuntime.svelte.ts @@ -56,7 +56,10 @@ import { selectPreviewTabsToClose, whereIs } from './sessionPreviewTabs.svelte' +import { pageItemKindLabel } from './previewPaths' import { + pageItemForLocation, + pageItemLocation, parsePreviewItemRoute, previewLocationContext, previewLocationLabel, @@ -96,7 +99,13 @@ import type { RawAppDomResult } from '$lib/components/raw_apps/rawAppDom' import { getNonStreamingMetadataCompletion } from '$lib/components/copilot/lib' -import { pendingUserAction, type DisplayMessage } from '$lib/components/copilot/chat/shared' +import { + pendingUserAction, + type ChatJob, + type DisplayMessage +} from '$lib/components/copilot/chat/shared' +import { readStoredChat } from '$lib/components/copilot/chat/HistoryManager.svelte' +import { getCurrentUserEmail } from '$lib/userScopedStorage' import type { ChatCompletionMessageParam } from 'openai/resources/index.mjs' // Per-kind load state for a session's editor target. Pure state container the @@ -128,6 +137,9 @@ export interface RawAppRuntimeValue { path: string custom_path?: string draft_path?: string + /** The app_version this cell's content forked from, carried so the editor's deploy + * guard and diff drawer compare the same pair the full-page editor does. */ + parent_version?: number } // The deployed baseline a raw-app cell diffs against (topbar Diff drawer). export interface RawAppSavedValue { @@ -143,6 +155,9 @@ export interface RawAppSavedValue { /** No deployed counterpart (draft-only); disables the topbar Diff. */ no_deployed?: boolean custom_path?: string + /** The deployed head at load time, which the editor's deploy guard falls back to when + * the draft carries no base of its own. */ + deployed_version?: number } // One editor cell per (kind, path) the session loads: the load slot plus the @@ -381,7 +396,8 @@ function createRuntime(session: Session): SessionRuntime { // What the side panel is showing, stamped on each user message so the chat // knows the page (and the row whose drawer is open) without spending a // get_preview_status round-trip. Live editors are skipped: they register - // themselves as the ACTIVE EDITOR through UserDraft's live-draft registry. + // themselves as the ACTIVE EDITOR through UserDraft's live-draft registry. A page + // item tab is not one of them, and reads as its list page with the item open. manager.activePreviewResolver = () => { const owner = getRuntime(session.id)?.previewTabs // What is on screen, not merely which tab is selected: the rule tells the model @@ -389,7 +405,8 @@ function createRuntime(session: Session): SessionRuntime { // point those at a page the user cannot see. const tab = owner?.displayedTab if (!tab) return undefined - if (resolvePreviewTab(tab.url).kind !== 'iframe') return undefined + const slotKind = resolvePreviewTab(tab.url).kind + if (slotKind !== 'iframe' && slotKind !== 'pageitem') return undefined return previewLocationContext(whereIs(tab)) } // Pre-flight: materialise the (still-transient) session, then commit @@ -587,8 +604,9 @@ function createRuntime(session: Session): SessionRuntime { // when the path has never been deployed. const aiDraft = UserDraft.get('flow', path, { workspace }) - // getDraft=true omits version_id (the plain getFlowByPath has it) — - // stamp it on so the flow doesn't always diff. Best-effort. + // Fallback for the head: the payload fetches below carry `version_id`, and + // this one covers a response that does not. Best-effort, and a request of + // its own, so the same response wins wherever both are available. let deployedVersionId: number | undefined try { deployedVersionId = (await FlowService.getFlowByPath({ workspace, path }))?.version_id @@ -602,12 +620,16 @@ function createRuntime(session: Session): SessionRuntime { // yet on the backend — draft-only flows are a valid state. try { const result = await FlowService.getFlowByPath({ workspace, path, getDraft: true }) - saved.val = result as SavedFlow + // The editor's deploy guard compares against the head, so keep this + // response's own and fall back to the one fetched above. + saved.val = { + ...(result as SavedFlow), + version_id: (result as SavedFlow).version_id ?? deployedVersionId + } } catch { saved.val = undefined } await initFlow(aiDraft, store, stateStore, workspace) - if (deployedVersionId != null && store.val) store.val.version_id = deployedVersionId slot.loadedPath = path slot.loadedWorkspace = workspace return @@ -615,8 +637,12 @@ function createRuntime(session: Session): SessionRuntime { // No local draft yet — seed from `result.draft ?? result`. const result = await FlowService.getFlowByPath({ workspace, path, getDraft: true }) - saved.val = result as SavedFlow - const flow: Flow = ((result as SavedFlow).draft ?? (result as Flow)) as Flow + saved.val = { + ...(result as SavedFlow), + version_id: (result as SavedFlow).version_id ?? deployedVersionId + } + const serverDraft = (result as SavedFlow).draft as Flow | undefined + const flow: Flow = (serverDraft ?? (result as Flow)) as Flow // Seed the per-tab last_sync from the server draft's timestamp so the // seeding save below attaches a matching last_sync and the server can // reject stale writes (see loadRawApp). Without this a server draft — @@ -629,7 +655,13 @@ function createRuntime(session: Session): SessionRuntime { ) UserDraft.save('flow', path, flow, { workspace }) await initFlow(flow, store, stateStore, workspace) - if (deployedVersionId != null && store.val) store.val.version_id = deployedVersionId + // A draft keeps the base it forked from, unknown included (it then falls + // back to the timestamps); only a fresh checkout takes the head, which is + // also what keeps it from always diffing. See loadScript. The head comes + // from the response that supplied the payload, so a deploy landing between + // the two requests cannot label this checkout as forked from the older one. + const head = (result as SavedFlow).version_id ?? deployedVersionId + if (head != null && store.val && !serverDraft) store.val.version_id = head slot.loadedPath = path slot.loadedWorkspace = workspace } catch (err) { @@ -672,11 +704,10 @@ function createRuntime(session: Session): SessionRuntime { } // Clone before layering the AI draft on top, else we'd mutate // `saved.val` in place and lose the pristine diff baseline. + const savedDraft = saved.val?.draft as NewScript | undefined const baseline: NewScript = saved.val ? (structuredClone( - $state.snapshot( - (saved.val.draft as NewScript | undefined) ?? (saved.val as NewScript) - ) + $state.snapshot(savedDraft ?? (saved.val as NewScript)) ) as NewScript) : { // Seed from the draft's own path (a rename lives in `draft_path`, @@ -694,9 +725,8 @@ function createRuntime(session: Session): SessionRuntime { schema: emptySchema(), language: (aiDraft.language ?? 'bun') as any } - if (saved.val?.hash) { - baseline.parent_hash = saved.val.hash - } + // Only a fresh checkout forks from the head; see the branch below. + if (!savedDraft && saved.val?.hash) baseline.parent_hash = saved.val.hash baseline.content = aiDraft.content if (aiDraft.language) baseline.language = aiDraft.language if (aiDraft.summary !== undefined) baseline.summary = aiDraft.summary @@ -711,10 +741,13 @@ function createRuntime(session: Session): SessionRuntime { saved.val = result as SavedScript // Clone before mutating, else `baseline` aliases `result` and // `baseline.parent_hash` corrupts the diff baseline. - const baseline = structuredClone( - ((result as SavedScript).draft as NewScript | undefined) ?? (result as NewScript) - ) - baseline.parent_hash = result.hash + const serverDraft = (result as SavedScript).draft as NewScript | undefined + const baseline = structuredClone(serverDraft ?? (result as NewScript)) + // Only a fresh checkout forks from the head. A draft keeps the base it has, + // unknown included: `draft.base` is derived from `parent_hash` on every + // save, so stamping the head over it would say this draft is up to date + // when it is not. An unknown base falls back to the timestamps. + if (!serverDraft) baseline.parent_hash = result.hash // Seed the per-tab last_sync from the server draft's timestamp so the // seeding save below attaches a matching last_sync and the server can // reject stale writes (see loadRawApp). Without this a server draft — @@ -775,7 +808,10 @@ function createRuntime(session: Session): SessionRuntime { path: result.path, policy: result.policy, custom_path: result.custom_path, - no_deployed: result.no_deployed + no_deployed: result.no_deployed, + deployed_version: Array.isArray(result.versions) + ? result.versions[result.versions.length - 1] + : undefined } } catch { saved.val = undefined @@ -811,7 +847,10 @@ function createRuntime(session: Session): SessionRuntime { path: result.path, policy: result.policy, custom_path: result.custom_path, - no_deployed: result.no_deployed + no_deployed: result.no_deployed, + deployed_version: Array.isArray(result.versions) + ? result.versions[result.versions.length - 1] + : undefined } // Prefer the server draft over the deployed value (mirrors the // flow/script `result.draft ?? result`). A raw-app draft is already @@ -841,7 +880,14 @@ function createRuntime(session: Session): SessionRuntime { summary: draftValue?.summary ?? result.summary ?? '', path: result.path, custom_path: draftValue?.custom_path ?? result.custom_path, - draft_path: draftValue?.draft_path + draft_path: draftValue?.draft_path, + // Only a fresh checkout forks from the head; a draft keeps its own base, + // unknown included, or it would read as up to date. See loadScript. + parent_version: draftValue + ? draftValue.parent_version + : Array.isArray(result.versions) + ? result.versions[result.versions.length - 1] + : undefined } // Seed the per-tab last_sync from the server draft's timestamp so // later saves attach a matching last_sync and the server can reject @@ -945,7 +991,7 @@ function createRuntime(session: Session): SessionRuntime { async function initRuntime(runtime: SessionRuntime, session: Session) { const { manager } = runtime - await manager.historyManager.init() + await manager.historyManager.init(session) manager.historyManager.setSessionId(session.id) // Restore linked files persisted for this session (live handles re-grant on send; // snapshots restore directly). Non-transient sessions persist immediately. @@ -960,7 +1006,9 @@ async function initRuntime(runtime: SessionRuntime, session: Session) { if (session.chatId) { manager.historyManager.setCurrentChatId(session.chatId) await manager.historyManager.tagChatWithSession(session.chatId, session.id) - await manager.loadPastChat(session.chatId) + // preserveQueue: restoring a session's own chat is not a conversation switch. + // A message queued while the restore ran is unsent user input waiting for it. + await manager.loadPastChat(session.chatId, { preserveQueue: true }) // loadPastChat only seeds the mask when the chat exists in history; a chatId // pointing at a chat not yet persisted (no turn saved) would leave it // undefined, and the Edits surface would then show every workspace draft. @@ -980,7 +1028,11 @@ export function getOrCreateRuntime(session: Session): SessionRuntime { if (!runtime) { runtime = createRuntime(session) runtimes.set(session.id, runtime) - initRuntime(runtime, session).catch((e) => console.error('Failed to init session runtime', e)) + // The gate opens even when the restore fails: a session that cannot read its + // history is still usable, and a send that waits forever is not. + runtime.manager.setReadyGate( + initRuntime(runtime, session).catch((e) => console.error('Failed to init session runtime', e)) + ) } return runtime } @@ -991,6 +1043,9 @@ export function disposeRuntime(sessionId: string) { runtime.manager.cancel('runtime disposed') runtime.manager.historyManager.close() runtimes.delete(sessionId) + // The sidebar reads the stored chat again rather than trust a read that + // predates everything this runtime saw. + peeks.delete(sessionId) } export function listRuntimes(): SessionRuntime[] { @@ -1024,7 +1079,13 @@ onRemoteTurnEnd((sessionId, chatId) => { async function applyRemoteTurnEnd(sessionId: string, chatId: string): Promise { const runtime = runtimes.get(sessionId) - if (!runtime) return + if (!runtime) { + // The driving tab may have rotated to a new chat; follow it, or the next + // read and a later runtime would open the old one. + setSessionChatId(sessionId, chatId) + refreshSessionChatPeek(sessionId) + return + } const m = runtime.manager // Two transient states get a short retry rather than a skip, because the // composer unlocks when this promise settles and a skip would unlock it on @@ -1066,7 +1127,9 @@ async function applyRemoteTurnEnd(sessionId: string, chatId: string): Promise { // open_page dispatches here to show a workspace page (Runs/Schedules) as a page // tab in the calling session's preview panel. Returns undefined when there is no // session so open_page can fall back to browser navigation. -setOpenPagePreviewHandler(({ sessionId: callerSessionId, href, label, newTab }) => { +setOpenPagePreviewHandler(({ sessionId: callerSessionId, href, label: pageLabel, newTab }) => { const sessionId = callerSessionId ?? sessionState.currentSessionId if (!sessionId) return undefined const session = sessionState.sessions.find((s) => s.id === sessionId) if (!session) return undefined const owner = getOrCreateRuntime(session).previewTabs + // A page opened on one item is that item's tab, and the report has to name what opened. + const pageItem = pageItemForLocation(href) + const label = pageItem + ? `the ${pageItemKindLabel(pageItem).toLowerCase()} ${promptSafe(pageItem.path)}` + : pageLabel // open() owns the whole decision — which tab already shows this page, whether the // requested view differs from what it shows, and whether a forced load is needed to // re-fire a drawer. Deciding any of that again here means two predicates for one @@ -1358,3 +1426,87 @@ export function getSessionChatStatus(runtime: SessionRuntime): SessionChatStatus if (last && (last.role === 'assistant' || last.role === 'tool')) return 'awaiting-user' return 'idle' } + +// --------------------------------------------------------------------------- +// Sessions without a runtime +// --------------------------------------------------------------------------- + +// A runtime costs a chat manager and its loaded transcript, and lives until its +// session is deleted, so the sidebar does not create one per listed session. It +// reads the session's one stored chat instead, once. Without a runtime nothing +// runs in this tab, so only two facts of that chat can matter: its length (for +// the unread count) and whether its last message failed. Every other status is +// a live one. +export interface SessionChatPeek { + status: 'error' | 'idle' + messageCount: number +} + +const peeks = new SvelteMap() +const peeksInFlight = new Set() +const peeksPendingRefresh = new Set() + +// Not gated on `detached`: a chat saved mid-wait stores its job undetached, and +// loadPastChat resumes polling every unfinished job either way. +function isLiveJob(j: ChatJob): boolean { + return ( + j.status === 'queued' || + j.status === 'running' || + j.status === 'suspended' || + j.status === 'scheduled' + ) +} + +function sessionById(sessionId: string): Session | undefined { + return sessionState.sessions.find((s) => s.id === sessionId) +} + +function refreshSessionChatPeek(sessionId: string): void { + peeks.delete(sessionId) + if (peeksInFlight.has(sessionId)) { + peeksPendingRefresh.add(sessionId) + return + } + const session = sessionById(sessionId) + if (session) void ensureSessionChatPeek(session) +} + +export function getSessionChatPeek(sessionId: string): SessionChatPeek | undefined { + return peeks.get(sessionId) +} + +/** Read the stored chat of a session that has no runtime. A chat with a detached + * job still running gets its runtime instead: only a runtime polls the job and + * resumes the conversation when it finishes. */ +export async function ensureSessionChatPeek(session: Session): Promise { + const id = session.id + const chatId = session.chatId + if (runtimes.has(id) || peeks.has(id) || peeksInFlight.has(id) || !chatId) return + const email = getCurrentUserEmail() + if (!email) return + peeksInFlight.add(id) + try { + const chat = await readStoredChat(chatId, email) + if (sessionById(id)?.chatId !== chatId) { + refreshSessionChatPeek(id) + return + } + if (!chat || runtimes.has(id)) return + if (chat.backgroundJobs?.some(isLiveJob)) { + getOrCreateRuntime(session) + return + } + const last = chat.displayMessages[chat.displayMessages.length - 1] + peeks.set(id, { + status: last?.role === 'user' && last.error ? 'error' : 'idle', + messageCount: chat.displayMessages.length + }) + } catch (e) { + console.error('Failed to read session chat', e) + } finally { + peeksInFlight.delete(id) + if (peeksPendingRefresh.delete(id)) { + refreshSessionChatPeek(id) + } + } +} diff --git a/frontend/src/lib/components/sessions/sessionState.svelte.ts b/frontend/src/lib/components/sessions/sessionState.svelte.ts index 3014e6d5e7..c5a2f28455 100644 --- a/frontend/src/lib/components/sessions/sessionState.svelte.ts +++ b/frontend/src/lib/components/sessions/sessionState.svelte.ts @@ -82,7 +82,9 @@ export type SessionSummarySource = 'placeholder' | 'generated' | 'manual' export type Session = { id: string - name: string + // Per-browser `session-N` label that older sessions-page URLs carried; only read + // to resolve those links. + name?: string // Committed strictly at first user-message send. Undefined for drafts // that have never been sent — those scope by `pending_workspace_id` // instead and don't show the fork bar. @@ -1005,8 +1007,8 @@ if (BROWSER) { }) } -export function findSessionByName(name: string): Session | undefined { - return sessionState.sessions.find((s) => s.name === name) +export function sessionPageHref(id: string): string { + return `/sessions?session=${encodeURIComponent(id)}` } // Bumped to ask the active session's composer to re-focus even when @@ -1045,15 +1047,6 @@ export function findEmptyLandingSession(): Session | undefined { ) } -// Session names are a per-browser counter (`session-N`) that the sessions page puts in -// its URL, so a new or restored record takes the number after the highest in use. -function nextSessionNumber(sessions: Session[]): number { - const numbers = sessions - .map((s) => /^session-(\d+)$/.exec(s.name)?.[1]) - .map((n) => (n ? parseInt(n, 10) : 0)) - return (numbers.length ? Math.max(...numbers) : 0) + 1 -} - export function createSession(): Session { // Reuse an existing untouched draft from the active family rather than pile a // blank entry on every `+`, so several pending sessions can still be built up @@ -1074,7 +1067,6 @@ export function createSession(): Session { return reusable } sessionState.sessions = sessionState.sessions.filter((s) => !isDiscardableDraft(s)) - const next = nextSessionNumber(sessionState.sessions) // Start in the workspace you're in. The one exception: a root you can't // deploy to (locked, no bypass) steers to its dev, since a session there // couldn't edit anything. The picker lets you switch. @@ -1100,7 +1092,6 @@ export function createSession(): Session { const summary = `${adj.charAt(0).toUpperCase() + adj.slice(1)} session` const session: Session = { id: createLongHash(), - name: `session-${next}`, summary, summarySource: 'placeholder', pending_workspace_id: pending && pending.length > 0 ? pending : undefined, @@ -1459,11 +1450,10 @@ export async function importSessions(records: Session[], email: string): Promise try { const tx = db.transaction('sessions', 'readwrite') const existing = new Set((await tx.store.getAllKeys()).map(String)) - let next = nextSessionNumber([...(await tx.store.getAll()), ...sessionState.sessions]) const restoredAt = Date.now() for (const r of records) { if (existing.has(r.id) || deletedSessionIds.has(r.id)) continue - const record: Session = { ...r, name: `session-${next++}`, restoredAt } + const record: Session = { ...r, restoredAt } delete record.transient delete record.workspace_root_id ensureSessionRootId(record) diff --git a/frontend/src/lib/components/sessions/sessionStateIndexedDb.test.ts b/frontend/src/lib/components/sessions/sessionStateIndexedDb.test.ts index dddfe795f7..fe0ad84d78 100644 --- a/frontend/src/lib/components/sessions/sessionStateIndexedDb.test.ts +++ b/frontend/src/lib/components/sessions/sessionStateIndexedDb.test.ts @@ -814,33 +814,28 @@ describe('sessionState IndexedDB persistence', () => { await vi.waitFor(() => expect(sessionState.sessions).toEqual([])) }) - // A restored backup must never replace what this browser has, come back after the - // user deleted it here, or take a name the sessions page already routes by. - it('importSessions adds only unknown, undeleted records under fresh names', async () => { + // A restored backup must never replace what this browser has, or come back after the + // user deleted it here. + it('importSessions adds only unknown, undeleted records', async () => { const user = freshUser() await login(user) - const local = session({ id: 'local', name: 'session-3', createdAt: 1, summary: 'mine' }) + const local = session({ id: 'local', createdAt: 1, summary: 'mine' }) await putSession(local) sessionState.sessions.push(local) deleteSession('local') await flush() - await putSession(session({ id: 'kept', name: 'session-5', createdAt: 2, summary: 'kept' })) + await putSession(session({ id: 'kept', createdAt: 2, summary: 'kept' })) const imported = await importSessions( [ - session({ id: 'local', name: 'session-1', createdAt: 1, summary: 'remote copy' }), - session({ id: 'kept', name: 'session-1', createdAt: 2, summary: 'remote copy' }), - session({ id: 'new', name: 'session-1', createdAt: 3, workspace_id: 'ws' }) + session({ id: 'local', createdAt: 1, summary: 'remote copy' }), + session({ id: 'kept', createdAt: 2, summary: 'remote copy' }), + session({ id: 'new', createdAt: 3, workspace_id: 'ws' }) ], user.email ) expect(imported).toEqual(['new']) - await vi.waitFor(() => - expect(sessionState.sessions.map((s) => [s.id, s.name])).toEqual([ - ['new', 'session-6'], - ['kept', 'session-5'] - ]) - ) + await vi.waitFor(() => expect(sessionState.sessions.map((s) => s.id)).toEqual(['new', 'kept'])) expect(sessionState.sessions.find((s) => s.id === 'kept')?.summary).toBe('kept') // The wrong user's name gets nothing written. diff --git a/frontend/src/lib/components/sessions/sessionSwitch.svelte.ts b/frontend/src/lib/components/sessions/sessionSwitch.svelte.ts index a7a492b874..36a91ee453 100644 --- a/frontend/src/lib/components/sessions/sessionSwitch.svelte.ts +++ b/frontend/src/lib/components/sessions/sessionSwitch.svelte.ts @@ -4,6 +4,7 @@ import { workspaceStore } from '$lib/stores' import { createSession, selectSession, + sessionPageHref, sessionInCurrentFamily, sessionLastActivityAt, sessionState, @@ -81,7 +82,7 @@ function resumableSession(): Session | undefined { export async function enterSessionMode(opts?: { replace?: boolean }): Promise { const target = resumableSession() ?? createSession() selectSession(target.id) - await goto(`/sessions?session_name=${encodeURIComponent(target.name)}`, { + await goto(sessionPageHref(target.id), { replaceState: opts?.replace ?? false }) } @@ -197,7 +198,7 @@ async function openInSession( navRouteOffered = true } selectSession(session.id) - await goto(`/sessions?session_name=${encodeURIComponent(session.name)}`) + await goto(sessionPageHref(session.id)) } // Open an editor's own hand-off, running its `beforeOpen` (which persists the @@ -243,5 +244,5 @@ export async function startSessionWithPrompt( // An empty prompt has nothing to send; leave the composer focused instead. if (opts?.autoSend && prompt.trim()) setSessionAutoSend(session.id) selectSession(session.id) - await goto(`/sessions?session_name=${encodeURIComponent(session.name)}`) + await goto(sessionPageHref(session.id)) } diff --git a/frontend/src/lib/components/sessions/sessionSwitch.test.ts b/frontend/src/lib/components/sessions/sessionSwitch.test.ts index 23ca52a693..d550ad8df4 100644 --- a/frontend/src/lib/components/sessions/sessionSwitch.test.ts +++ b/frontend/src/lib/components/sessions/sessionSwitch.test.ts @@ -62,7 +62,7 @@ describe('enterSessionMode — restore is scoped to the active family', () => { try { await enterSessionMode() expect(sessionState.currentSessionId).toBe('sw-in-family') - expect(goto).toHaveBeenCalledWith('/sessions?session_name=session-911', { + expect(goto).toHaveBeenCalledWith('/sessions?session=sw-in-family', { replaceState: false }) } finally { @@ -82,7 +82,7 @@ describe('enterSessionMode — restore is scoped to the active family', () => { try { await enterSessionMode() expect(sessionState.currentSessionId).toBe('sw-local') - expect(goto).toHaveBeenCalledWith('/sessions?session_name=session-913', { + expect(goto).toHaveBeenCalledWith('/sessions?session=sw-local', { replaceState: false }) } finally { diff --git a/frontend/src/lib/components/sessions/sessionUnread.svelte.ts b/frontend/src/lib/components/sessions/sessionUnread.svelte.ts index 281c6571bc..26ce3f205b 100644 --- a/frontend/src/lib/components/sessions/sessionUnread.svelte.ts +++ b/frontend/src/lib/components/sessions/sessionUnread.svelte.ts @@ -1,4 +1,4 @@ -import type { SessionRuntime } from './sessionRuntime.svelte' +import type { SessionChatPeek, SessionRuntime } from './sessionRuntime.svelte' import { sessionState, putSession } from './sessionState.svelte' // Per-session "last seen" watermark — the displayMessages count the last time @@ -20,11 +20,16 @@ export function markSessionSeen(sessionId: string, count: number) { void putSession(s) } -// Number of unread messages for a session. Undefined / unloaded runtime -// returns 0 — until messages are hydrated we don't know what's new. -export function unreadCountFor(sessionId: string, runtime: SessionRuntime | undefined): number { - if (!runtime) return 0 +// Number of unread messages for a session, from its runtime when it has one, else +// from the sidebar's read of its stored chat. Neither loaded returns 0 — until +// messages are read we don't know what's new. +export function unreadCountFor( + sessionId: string, + runtime: SessionRuntime | undefined, + peek: SessionChatPeek | undefined +): number { + const total = runtime ? runtime.manager.displayMessages.length : peek?.messageCount + if (total === undefined) return 0 const seen = sessionState.sessions.find((x) => x.id === sessionId)?.lastSeenCount ?? 0 - const total = runtime.manager.displayMessages.length return Math.max(0, total - seen) } diff --git a/frontend/src/lib/components/sessions/useUserDraftSync.svelte.ts b/frontend/src/lib/components/sessions/useUserDraftSync.svelte.ts index 7e89d3b945..64f7f7e6c2 100644 --- a/frontend/src/lib/components/sessions/useUserDraftSync.svelte.ts +++ b/frontend/src/lib/components/sessions/useUserDraftSync.svelte.ts @@ -94,7 +94,10 @@ export function useUserDraftSync(opts: UserDraftSyncOptions): void // re-apply only advances the sig, never reverts an edit or fires a save. if (sig === lastInboundSig) return lastInboundSig = sig - codec.applyDraftToStore(incoming) + // Copies both ways: a store sharing proxies with the draft makes every local + // edit re-fire this effect as if a draft came in, reassigning the store on + // each keystroke. + codec.applyDraftToStore($state.snapshot(incoming) as Draft) }) }) @@ -125,7 +128,9 @@ export function useUserDraftSync(opts: UserDraftSyncOptions): void untrack(() => { const current = UserDraft.get(codec.itemKind, path, { workspace }) if (current && codec.sig(current) === sig) return - const toSave = codec.storeToDraft(current) ?? draft + const toSave = $state.snapshot(codec.storeToDraft(current) ?? draft) as Draft + // Our own write comes back through the handle; the inbound side must skip it. + lastInboundSig = codec.sig(toSave) UserDraft.save(codec.itemKind, path, toSave, { workspace }) }) } diff --git a/frontend/src/lib/components/settings/CreateToken.svelte b/frontend/src/lib/components/settings/CreateToken.svelte index 146f844b3f..8ee5391443 100644 --- a/frontend/src/lib/components/settings/CreateToken.svelte +++ b/frontend/src/lib/components/settings/CreateToken.svelte @@ -1,11 +1,6 @@ {#if captureInfo} diff --git a/frontend/src/lib/components/triggers/email/EmailTriggerEditorConfigSection.svelte b/frontend/src/lib/components/triggers/email/EmailTriggerEditorConfigSection.svelte index 3a60d3be71..4a44e01b30 100644 --- a/frontend/src/lib/components/triggers/email/EmailTriggerEditorConfigSection.svelte +++ b/frontend/src/lib/components/triggers/email/EmailTriggerEditorConfigSection.svelte @@ -2,8 +2,6 @@ import { Alert } from '$lib/components/common' import Required from '$lib/components/Required.svelte' import Section from '$lib/components/Section.svelte' - import { userStore, workspaceStore } from '$lib/stores' - import { getTriggerWorkspace } from '$lib/components/triggers/triggerWorkspace' // import { page } from '$app/state' import { getEmailAddress, getEmailDomain } from './utils' import { isCloudHosted } from '$lib/cloud' @@ -12,6 +10,10 @@ import { untrack } from 'svelte' import { EmailTriggerService } from '$lib/gen' import ClipboardPanel from '$lib/components/details/ClipboardPanel.svelte' + import { + useOperatingUser, + useOperatingWorkspace + } from '$lib/components/operatingWorkspace.svelte' interface Props { initialTriggerPath?: string | undefined dirtyLocalPart?: boolean @@ -35,8 +37,10 @@ isDraftOnly = true, showTestingBadge = false }: Props = $props() - const triggerWs = getTriggerWorkspace() - const wsId = $derived(triggerWs?.() ?? $workspaceStore) + const operatingWorkspace = useOperatingWorkspace() + const operatingUser = useOperatingUser() + const actingUser = $derived(operatingUser.current) + const wsId = $derived($operatingWorkspace) let validateTimeout: number | undefined = undefined @@ -95,7 +99,7 @@ local_part === undefined && (local_part = '') }) - let userIsAdmin = $derived($userStore?.is_admin || $userStore?.is_super_admin) + let userIsAdmin = $derived(actingUser?.is_admin || actingUser?.is_super_admin) let userCanEditConfig = $derived(userIsAdmin || isDraftOnly) // User can edit config if they are admin or if the trigger is a draft which will not be saved diff --git a/frontend/src/lib/components/triggers/email/EmailTriggerEditorInner.svelte b/frontend/src/lib/components/triggers/email/EmailTriggerEditorInner.svelte index d2133cf865..30f848004f 100644 --- a/frontend/src/lib/components/triggers/email/EmailTriggerEditorInner.svelte +++ b/frontend/src/lib/components/triggers/email/EmailTriggerEditorInner.svelte @@ -2,6 +2,7 @@ import { Button } from '$lib/components/common' import { clearPageDrawerAnchor, + handOffPageDrawer, setPageDrawerAnchor } from '$lib/components/sessions/pageDrawerSession' import { TRIGGER_PAGES } from '$lib/components/sessions/previewPaths' @@ -17,8 +18,7 @@ type Retry, type TriggerMode } from '$lib/gen' - import { usedTriggerKinds, userStore, workspaceStore } from '$lib/stores' - import { getTriggerWorkspace } from '$lib/components/triggers/triggerWorkspace' + import { usedTriggerKinds } from '$lib/stores' import { canWrite, capitalize, emptyString, sendUserToast } from '$lib/utils' import Section from '$lib/components/Section.svelte' import { Loader2 } from 'lucide-svelte' @@ -38,9 +38,16 @@ import LocalDraftBanner from '$lib/components/LocalDraftBanner.svelte' import TriggerSuspendedJobsAlert from '../TriggerSuspendedJobsAlert.svelte' import TriggerSuspendedJobsModal from '../TriggerSuspendedJobsModal.svelte' + import { + useOperatingUser, + useOperatingWorkspace, + useOperatingWorkspaceHref + } from '$lib/components/operatingWorkspace.svelte' let { useDrawer = true, + inline = false, + onClose = undefined, hideTarget = false, description = undefined, isEditor = false, @@ -55,8 +62,11 @@ trigger = undefined, customSaveBehavior = undefined } = $props() - const triggerWs = getTriggerWorkspace() - const wsId = $derived(triggerWs?.() ?? $workspaceStore) + const operatingWorkspace = useOperatingWorkspace() + const operatingUser = useOperatingUser() + const actingUser = $derived(operatingUser.current) + const operatingHref = useOperatingWorkspaceHref() + const wsId = $derived($operatingWorkspace) // Form data state let initialPath = $state('') @@ -75,7 +85,13 @@ let workspaced_local_part = $state(false) let drawerLoading = $state(true) let showLoader = $state(false) - let can_write = $state(true) + let permsPath = $state(undefined) + let permsForWrite = $state | undefined>(undefined) + // The acting user in the operating workspace arrives asynchronously, and an unknown user + // refuses — so the editor stays read-only until the lookup lands, which is the safe answer. + const can_write = $derived( + permsPath === undefined ? true : canWrite(permsPath, permsForWrite ?? {}, actingUser) + ) let extraPerms = $state | undefined>(undefined) let error_handler_path: string | undefined = $state() let error_handler_args: Record = $state({}) @@ -95,7 +111,7 @@ let originalConfig = $state(undefined) let hasChanged = $derived(!deepEqual(getEmailTriggerConfig(), originalConfig ?? {})) - const isAdmin = $derived($userStore?.is_admin || $userStore?.is_super_admin) + const isAdmin = $derived(actingUser?.is_admin || actingUser?.is_super_admin) const emailConfig = $derived.by(getEmailTriggerConfig) const draftSync = useTriggerDraftSync({ @@ -127,6 +143,9 @@ defaultConfig?: Partial, fixedScriptPath_?: string ) { + if (handOffPageDrawer(TRIGGER_PAGES.email.path, ePath)) return + // A `whoami` that failed earlier would otherwise pin this workspace to "unknown user". + operatingUser.forgetFailures() drawerLoading = true let loader = setTimeout(() => { showLoader = true @@ -206,6 +225,8 @@ } function loadTriggerConfig(cfg?: Partial): void { + // The loaded trigger says what it runs; an opener's `isFlow` is only its guess. + if (cfg?.is_flow !== undefined) itemKind = cfg.is_flow ? 'flow' : 'script' script_path = cfg?.script_path ?? '' initialScriptPath = cfg?.script_path ?? '' is_flow = cfg?.is_flow ?? false @@ -213,7 +234,8 @@ local_part = cfg?.local_part ?? '' workspaced_local_part = cfg?.workspaced_local_part ?? false extraPerms = cfg?.extra_perms ?? undefined - can_write = canWrite(path, cfg?.extra_perms ?? {}, $userStore) + permsPath = path + permsForWrite = cfg?.extra_perms ?? {} error_handler_path = cfg?.error_handler_path error_handler_args = cfg?.error_handler_args ?? {} retry = cfg?.retry @@ -265,7 +287,7 @@ saveCfg, edit, wsId!, - !!$userStore?.is_admin || !!$userStore?.is_super_admin, + !!actingUser?.is_admin || !!actingUser?.is_super_admin, usedTriggerKinds ) if (isSaved) { @@ -405,7 +427,7 @@ bind:scriptPath={script_path} {initialScriptPath} canWrite={can_write} - isOperator={!!$userStore?.operator} + isOperator={!!actingUser?.operator} promptClass="text-xs mt-3 mb-1 text-primary" > {#snippet createButton()} @@ -414,7 +436,9 @@ btnClasses="ml-4" variant="accent" size="xs" - href={itemKind === 'flow' ? '/flows/add?hub=72' : '/scripts/add?hub=hub%2F19813'} + href={operatingHref( + itemKind === 'flow' ? '/flows/add?hub=72' : '/scripts/add?hub=hub%2F19813' + )} target="_blank">Create from template {/if} @@ -486,36 +510,44 @@ {/if} {/snippet} -{#if useDrawer} +{#snippet drawerBody()} + (inline ? onClose?.() : drawer?.closeDrawer())} + > + {#snippet actions()} + {@render saveButton()} + {/snippet} + {#snippet banner()} + draftSync.deployed} + reserveSpace={draftSync.hasBaseline} + getCurrent={() => draftSync.current} + onDiscard={() => draftSync.resetToDeployed(initialPath)} + disabled={!can_write} + /> + {/snippet} + {@render config()} + +{/snippet} + +{#if useDrawer && inline} + {@render drawerBody()} +{:else if useDrawer} clearPageDrawerAnchor(TRIGGER_PAGES.email.path)} > - drawer?.closeDrawer()} - > - {#snippet actions()} - {@render saveButton()} - {/snippet} - {#snippet banner()} - draftSync.deployed} - reserveSpace={draftSync.hasBaseline} - getCurrent={() => draftSync.current} - onDiscard={() => draftSync.resetToDeployed(initialPath)} - disabled={!can_write} - /> - {/snippet} - {@render config()} - + {@render drawerBody()} {:else}
diff --git a/frontend/src/lib/components/triggers/email/EmailTriggerPanel.svelte b/frontend/src/lib/components/triggers/email/EmailTriggerPanel.svelte index 12ec403aeb..7eac6a8971 100644 --- a/frontend/src/lib/components/triggers/email/EmailTriggerPanel.svelte +++ b/frontend/src/lib/components/triggers/email/EmailTriggerPanel.svelte @@ -1,11 +1,15 @@
{#if resourceId} -
+
Selected: {resourceName || resourceId}
{:else} -
+
No file selected
@@ -198,21 +196,27 @@
- {/if} -
- {#if editable && row.id !== ADMIN_ROLE} - removeRole(row.id)} /> - {/if} -
- -
- {/each} -
- - {#if editable && unusedRoles.length > 0} -
- -
+ + Role + + admin is the connection the data table used before roles, so it owns every + existing object and cannot be removed. Every other role is a login defined for + the whole instance, with only the privileges granted to it under Access. + + + + Tenants + + Users, groups and folders allowed to connect as this role. Workspace admins can + use every role. + + + + Default + + The role a job gets when it names none — no `-- role` annotation, no `?role=` in + the reference. Callers still have to be one of its tenants. + + + + + + + {#each roles as role (roleKey(role))} + {@const isAdmin = role.id === ADMIN_DATATABLE_ROLE} + + +
+ {role.name ?? role.id} + {#if !role.name} + + no longer defined on this instance + + {:else if role.id === undefined} + + {#if $superadmin} +
+ Create it on the instance to use it here. + +
+ {:else} + Only a superadmin can create it on the instance. + {/if} + + {/if} +
+ + + item.group} + disabled={!editable} + placeholder="Nobody — add users, groups or folders" + /> + + +
+ { + if (role.id !== undefined) defaultRoleId = role.id + }} + /> +
+ + + {#if editable && !isAdmin} + removeRole(role)} /> + {/if} + + + {/each} + {#if editable} + + +
+
+ {/if} + + {#if info?.supported && !hasUnsavedChanges} +
+ +
+ {/if} {/if} + + {#snippet actions()} + {#if editable} + + {/if} + {/snippet} + +{#if $superadmin} + +{/if} diff --git a/frontend/src/lib/components/workspaceSettings/DataTableRolesSection.svelte b/frontend/src/lib/components/workspaceSettings/DataTableRolesSection.svelte index e46b578597..b863dac06b 100644 --- a/frontend/src/lib/components/workspaceSettings/DataTableRolesSection.svelte +++ b/frontend/src/lib/components/workspaceSettings/DataTableRolesSection.svelte @@ -3,7 +3,6 @@ import CloseButton from '../common/CloseButton.svelte' import TextInput from '../text_input/TextInput.svelte' import Toggle from '../Toggle.svelte' - import Tooltip from '../Tooltip.svelte' import ConfirmationModal from '../common/confirmationModal/ConfirmationModal.svelte' import { createAsyncConfirmationModal } from '../common/confirmationModal/asyncConfirmationModal.svelte' import Cell from '../table/Cell.svelte' @@ -14,11 +13,22 @@ import { SettingService, type InstanceDatatableRole } from '$lib/gen' import { sendUserToast } from '$lib/toast' + let { + initialName = '', + onChanged + }: { + /** Prefills the name of the role to add. */ + initialName?: string + /** Called after every change to the catalog, whether or not it went through. */ + onChanged?: () => void + } = $props() + let roles = $state([]) let loading = $state(true) let loadError = $state(undefined) let busy = $state(false) - let newName = $state('') + // svelte-ignore state_referenced_locally + let newName = $state(initialName) /** Which role's name is being edited, and to what. */ let renaming = $state<{ id: string; name: string } | undefined>(undefined) @@ -49,6 +59,7 @@ // holds, so a failed flip has to snap back rather than sit there claiming it landed. await load() busy = false + onChanged?.() } } @@ -83,16 +94,6 @@
-
-

Instance roles

- - A data table role is a real Postgres login on this instance, shared by every instance - database. A job that names one connects as it, and Postgres decides what it may touch — grant - it privileges with SQL. Which people may use a role on a given data table is set per data - table, in its roles drawer. - -
- {#if loadError} {loadError} {:else} diff --git a/frontend/src/lib/components/workspaceSettings/DataTableSettings.svelte b/frontend/src/lib/components/workspaceSettings/DataTableSettings.svelte index 867cc58d87..ebc4bd8a7f 100644 --- a/frontend/src/lib/components/workspaceSettings/DataTableSettings.svelte +++ b/frontend/src/lib/components/workspaceSettings/DataTableSettings.svelte @@ -64,12 +64,11 @@ + +{#if !hideTrigger} + +{/if} + + + drawer?.closeDrawer()} + tooltip="A data table role is a real Postgres login on this instance, shared by every instance database. A job that names one connects as it, and Postgres decides what it may touch. Which people may use a role on a given data table, and what it may do there, is set per data table, in its roles drawer." + > + {#snippet titleExtra()} + Beta + {/snippet} + {#key openCount} + + {/key} + + diff --git a/frontend/src/lib/components/workspaceSettings/ModelPricing.svelte b/frontend/src/lib/components/workspaceSettings/ModelPricing.svelte index 85824c1c95..5adc268474 100644 --- a/frontend/src/lib/components/workspaceSettings/ModelPricing.svelte +++ b/frontend/src/lib/components/workspaceSettings/ModelPricing.svelte @@ -4,7 +4,7 @@ import TextInput from '../text_input/TextInput.svelte' import { getKnownModelPrice, inheritedCacheRates } from '../copilot/modelPricing' import { modelKey } from '../copilot/modelConfig' - import { stripLegacyThinkingSuffix } from '../copilot/reasoningRegistry' + import { chatModelsByProvider } from './perModelSettings' import { ChevronDown, ChevronUp } from 'lucide-svelte' import { slide } from 'svelte/transition' import SettingCard from '../instanceSettings/SettingCard.svelte' @@ -25,28 +25,7 @@ let errors = $state>({}) let collapsedProviders = $state>({}) - // Rates are keyed by the id the chat reports usage under, and `setCopilotInfo` - // strips the deprecated `/thinking` suffix before the chat ever sees a model. A - // row built from the raw config would save an override under a key nothing - // reports, so it would sit in settings looking applied and never price anything. - // Stripping can collapse two configured slots onto one model, hence the dedupe. - const modelsByProvider = $derived( - Object.entries(aiProviders).reduce( - (acc, [provider, config]) => { - const seen = new Set() - acc[provider] = config.models.flatMap((configured) => { - const model = stripLegacyThinkingSuffix(configured) - if (seen.has(model)) { - return [] - } - seen.add(model) - return [{ provider: provider as AIProvider, model }] - }) - return acc - }, - {} as Record> - ) - ) + const modelsByProvider = $derived(chatModelsByProvider(aiProviders)) type Field = 'input' | 'output' | 'cache_read' | 'cache_write' type CacheField = Extract @@ -228,13 +207,13 @@ {#if overridden}
Overriding the built-in price - +
{/if} {#if errors[key]} diff --git a/frontend/src/lib/components/workspaceSettings/ModelTokenLimits.svelte b/frontend/src/lib/components/workspaceSettings/ModelTokenLimits.svelte index 7ad9034e3b..5b76430148 100644 --- a/frontend/src/lib/components/workspaceSettings/ModelTokenLimits.svelte +++ b/frontend/src/lib/components/workspaceSettings/ModelTokenLimits.svelte @@ -1,176 +1,158 @@ {#if Object.keys(aiProviders).length > 0} - +
- {#each Object.entries(modelsByProvider).filter(([provider, models]) => models.length > 0) as [provider, models]} + {#each Object.entries(modelsByProvider).filter(([_, models]) => models.length > 0) as [provider, models] (provider)} {@const isExpanded = !collapsedProviders[provider]} - {@const hasCustom = hasCustomSettings(provider, models)}
- + {#if isExpanded}
- {#each models as { model }} - {@const currentTokens = getCurrentTokensForModel(provider as AIProvider, model)} - {@const defaultTokens = getDefaultTokensForModel(provider as AIProvider, model)} - {@const isAtDefault = isModelAtDefault(provider as AIProvider, model)} + {#each models as { model } (model)} + {@const key = modelKey(provider as AIProvider, model)} + {@const fallback = getDefault(provider as AIProvider, model)} + {@const overridden = isOverridden(provider as AIProvider, model)}
{model}
- { - const value = parseInt(e.currentTarget.value) - if (!isNaN(value)) { - updateTokensForModel(provider as AIProvider, model, value) - } - }} - class="w-20 px-2 py-1 text-xs text-center border border-gray-200 dark:border-gray-700 rounded bg-surface focus:outline-none focus:ring-2 focus:ring-blue-500 focus:border-transparent" - /> +
+ { + if (e.currentTarget.value === '') { + resetModel(provider as AIProvider, model) + return + } + const value = Number(e.currentTarget.value) + if (!isNaN(value)) { + updateTokens(provider as AIProvider, model, value) + } + }, + onblur: (e: Event & { currentTarget: HTMLInputElement }) => { + // Resync a value the state refused, so what is shown is what is stored. + e.currentTarget.value = String(limits[key] ?? '') + errors[key] = '' + } + }} + /> +
tokens
- {#if !isAtDefault} -
- Default: {defaultTokens} tokens + {#if overridden} +
+ + {fallback.assumed ? 'Assumed' : 'Default'}: {fallback.tokens.toLocaleString()} + tokens +
- {#if errors[getModelKey(provider as AIProvider, model)]} -
{errors[getModelKey(provider as AIProvider, model)]}
- {/if} + {/if} + {#if errors[key]} +
{errors[key]}
{/if}
{/each} diff --git a/frontend/src/lib/components/workspaceSettings/NewDataTableMigrationModal.svelte b/frontend/src/lib/components/workspaceSettings/NewDataTableMigrationModal.svelte index d16f948b88..04cf841b28 100644 --- a/frontend/src/lib/components/workspaceSettings/NewDataTableMigrationModal.svelte +++ b/frontend/src/lib/components/workspaceSettings/NewDataTableMigrationModal.svelte @@ -8,12 +8,17 @@ import TextInput from '../text_input/TextInput.svelte' import SimpleEditor from '../SimpleEditor.svelte' import { WorkspaceService, type DatatableMigration } from '$lib/gen' + import { listUsableDatatableRoles } from '../datatableUsableRoles' import { sendUserToast } from '$lib/toast' import { tick } from 'svelte' import ConfirmationModal from '../common/confirmationModal/ConfirmationModal.svelte' import { createAsyncConfirmationModal } from '../common/confirmationModal/asyncConfirmationModal.svelte' import Portal from '$lib/components/Portal.svelte' import { fetchPendingMigrations, outOfOrderRunMessage } from './datatableMigrationUtils' + import Select from '../select/Select.svelte' + import { parseMigrationRole, withMigrationRole } from '../datatableMigrationRole' + import { ADMIN_DATATABLE_ROLE } from '../dbTypes' + import { resource } from 'runed' let { workspace, @@ -50,6 +55,8 @@ let tab = $state('up') let name = $state('') let nameInput = $state() + let upEditor = $state() + let downEditor = $state() // A valid migration name is non-empty and limited to letters, digits, '_' and '-'. const MIGRATION_NAME_RE = /^[a-zA-Z0-9_-]+$/ let nameInvalid = $derived(!MIGRATION_NAME_RE.test(name.trim())) @@ -60,6 +67,96 @@ const confirmationModal = createAsyncConfirmationModal() + // The role lives in the SQL as its `-- role ` annotation, so the code is the single + // source of truth and the Select is a view onto it: reading parses, writing rewrites the + // annotation. + const usableRoles = resource( + () => [workspace, datatable] as const, + async ([ws, dt]) => { + try { + return await listUsableDatatableRoles(ws, dt) + } catch (e) { + console.error('Failed to load data table roles:', e) + return null + } + } + ) + // Not a valid role name, so it cannot collide with one. + const NO_ROLE = '(no role)' + let declaredUp = $derived(parseMigrationRole(codeUp)) + let declaredDown = $derived(enableDown ? parseMigrationRole(codeDown) : undefined) + let malformedLine = $derived( + declaredUp.kind === 'malformed' + ? declaredUp.line + : declaredDown?.kind === 'malformed' + ? declaredDown.line + : undefined + ) + const roleOf = (d: typeof declaredUp) => (d.kind === 'role' ? d.role : undefined) + // A rollback runs as the role its own SQL names: under another role than the up migration it + // typically cannot touch what the up created. + let sqlProblem = $derived( + malformedLine !== undefined + ? malformedMessage(malformedLine) + : declaredDown !== undefined && roleOf(declaredDown) !== roleOf(declaredUp) + ? `The down migration runs as ${roleOf(declaredDown) ?? 'admin (no role)'} but the up migration as ${roleOf(declaredUp) ?? 'admin (no role)'}: make their role annotations match` + : undefined + ) + let selectedRole = $derived(declaredUp.kind === 'role' ? declaredUp.role : NO_ROLE) + let permissioned = $derived(!!usableRoles.current?.permissioned) + // No annotation runs as admin, which the server allows exactly to those who may use `admin`. + let adminUsable = $derived(!!usableRoles.current?.roles.includes(ADMIN_DATATABLE_ROLE)) + let roleItems = $derived.by(() => { + const usable = usableRoles.current + if (!usable?.permissioned) return [] + const names = usable.roles.filter((r) => r !== ADMIN_DATATABLE_ROLE) + // A role the SQL names but the caller cannot use is still shown, or the picker would + // misreport what the migration runs as. + if (declaredUp.kind === 'role' && !names.includes(declaredUp.role)) { + names.push(declaredUp.role) + } + const items = names.map((r) => ({ + value: r, + label: r === usable.default_role ? `${r} (default)` : r + })) + if (adminUsable || declaredUp.kind === 'none') { + items.push({ + value: NO_ROLE, + label: 'No role — runs as admin with full access' + }) + } + return items + }) + + function setRole(value: string | undefined) { + const role = value === NO_ROLE ? undefined : value + codeUp = withMigrationRole(codeUp, role) + // Up and down agree: a rollback run as another role could fail on objects it does not own. + if (enableDown) codeDown = withMigrationRole(codeDown, role) + // Assigning the bound value does not repaint the editor, and its next keystroke would write + // the stale text back. + upEditor?.setCode(codeUp) + if (enableDown) downEditor?.setCode(codeDown) + } + + // Set by `open` when the SQL names no role yet: the data table's default is written once its + // roles are known. + let applyDefaultRole = $state(false) + $effect(() => { + // `undefined` until the first answer lands; `null` when it failed. + const usable = usableRoles.current + if (!applyDefaultRole || !isOpen || usableRoles.loading || usable === undefined) return + applyDefaultRole = false + if (!usable?.permissioned || declaredUp.kind !== 'none') return + const role = + usable.default_role !== ADMIN_DATATABLE_ROLE && usable.roles.includes(usable.default_role) + ? usable.default_role + : usable.default_role === ADMIN_DATATABLE_ROLE && adminUsable + ? undefined + : usable.roles.find((r) => r !== ADMIN_DATATABLE_ROLE) + if (role !== undefined) setRole(role) + }) + // Frame the migration body in an explicit transaction so it applies atomically. function wrapInTransaction(body: string): string { return `BEGIN;\n\n${body}\n\nEND;` @@ -72,15 +169,35 @@ } const PLACEHOLDER = wrapInTransaction('-- Add your migration here') + function malformedMessage(line: string): string { + return `Malformed role annotation \`${line}\`: write it as \`-- role \`, or pick the role above` + } + export function open(prefill?: { name?: string; codeUp?: string; codeDown?: string }) { + // Roles and the default can have changed since the last open (the roles drawer sits next + // to this modal), and the default role is written from this answer. + usableRoles.refetch() name = prefill?.name ?? '' // Start from the transaction template; when prefilled from detected DDL, - // wrap that DDL in the same BEGIN; ... END; frame. - codeUp = prefill?.codeUp - ? wrapInTransaction(ensureTrailingSemicolon(prefill.codeUp)) - : PLACEHOLDER + // wrap that DDL in the same BEGIN; ... END; frame. A role the prefill declares is taken + // out first and put back on top: below `BEGIN;` it would not be read. + const prefillRole = prefill?.codeUp ? parseMigrationRole(prefill.codeUp) : undefined + if (prefill?.codeUp) { + const wrapped = wrapInTransaction( + ensureTrailingSemicolon(withMigrationRole(prefill.codeUp, undefined)) + ) + codeUp = + prefillRole?.kind === 'role' + ? withMigrationRole(wrapped, prefillRole.role) + : prefillRole?.kind === 'malformed' + ? `${prefillRole.line}\n${wrapped}` + : wrapped + } else { + codeUp = PLACEHOLDER + } codeDown = prefill?.codeDown ?? PLACEHOLDER enableDown = (prefill?.codeDown ?? '') !== '' + applyDefaultRole = prefillRole === undefined || prefillRole.kind === 'none' tab = 'up' isOpen = true // Focus the name field once the modal content has rendered. @@ -96,6 +213,10 @@ sendUserToast("Invalid migration name: use only letters, digits, '_' and '-'", true) return } + if (sqlProblem !== undefined) { + sendUserToast(sqlProblem, true) + return + } if (run) { // A new migration gets the highest timestamp, so any still-pending // migration is earlier: running only this one applies it out of order. @@ -176,29 +297,65 @@ closeOnOutsideClick={false} >
- +
+ + {#if permissioned} +
` — Specific table in public schema - `/:
` — Table in specific schema +**Roles:** when a datatable is under roles, its queries run as a role, which only reaches what it was granted. `roles` records the role the app uses each datatable through; the app's code must pass the same role: `wmill.datatable('main', { role: 'analyst' })` in TypeScript, `wmill.datatable('main', role='analyst')` in Python. A datatable without an entry is used as its default role. + ## SQL Migrations (sql_to_apply/) The `sql_to_apply/` folder is for creating/modifying database tables during development. diff --git a/system_prompts/base/raw-app-cli.md b/system_prompts/base/raw-app-cli.md index cd98e06ff5..030b958379 100644 --- a/system_prompts/base/raw-app-cli.md +++ b/system_prompts/base/raw-app-cli.md @@ -167,6 +167,8 @@ data: tables: - main/users # Table in public schema - main/app_schema:items # Table in specific schema + roles: # Optional: the role the app uses each datatable through + main: analyst ``` **Table reference formats:** @@ -174,6 +176,8 @@ data: - `/
` — Specific table in public schema - `/:
` — Table in specific schema +**Roles:** when a datatable is under roles, its queries run as a role, which only reaches what it was granted. `roles` records the role the app uses each datatable through; the app's code must pass the same role: `wmill.datatable('main', { role: 'analyst' })` in TypeScript, `wmill.datatable('main', role='analyst')` in Python. A datatable without an entry is used as its default role. + ## SQL Migrations (sql_to_apply/) The `sql_to_apply/` folder is for creating/modifying database tables during development. diff --git a/typescript-client/jsr.json b/typescript-client/jsr.json index e375863aa3..26441041a1 100644 --- a/typescript-client/jsr.json +++ b/typescript-client/jsr.json @@ -1,6 +1,6 @@ { "name": "@windmill/windmill", - "version": "1.815.0", + "version": "1.817.0", "exports": "./src/index.ts", "publish": { "exclude": ["!src", "./s3Types.ts", "./sqlUtils.ts", "./client.ts", "./wacError.ts"] diff --git a/typescript-client/package.json b/typescript-client/package.json index 49c63650e7..4badfa307b 100644 --- a/typescript-client/package.json +++ b/typescript-client/package.json @@ -1,7 +1,7 @@ { "name": "windmill-client", "description": "Windmill SDK client for browsers and Node.js", - "version": "1.815.0", + "version": "1.817.0", "author": "Ruben Fiszel", "license": "Apache 2.0", "homepage": "https://github.com/windmill-labs/windmill/tree/main/typescript-client#readme", diff --git a/version.txt b/version.txt index 297171ea20..2edd9ebec3 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -1.815.0 +1.817.0 diff --git a/windmill-yaml-validator/package-lock.json b/windmill-yaml-validator/package-lock.json index c65c508fe7..e07061650d 100644 --- a/windmill-yaml-validator/package-lock.json +++ b/windmill-yaml-validator/package-lock.json @@ -1,12 +1,12 @@ { "name": "windmill-yaml-validator", - "version": "1.815.0", + "version": "1.817.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "windmill-yaml-validator", - "version": "1.815.0", + "version": "1.817.0", "license": "Apache 2.0", "dependencies": { "@stoplight/yaml": "^4.3.0", diff --git a/windmill-yaml-validator/package.json b/windmill-yaml-validator/package.json index ec013fca2e..01bb378dcd 100644 --- a/windmill-yaml-validator/package.json +++ b/windmill-yaml-validator/package.json @@ -1,6 +1,6 @@ { "name": "windmill-yaml-validator", - "version": "1.815.0", + "version": "1.817.0", "description": "YAML validator for Windmill flow, schedule, and trigger files", "main": "dist/index.js", "types": "dist/index.d.ts",